def gb_parliament(): counter = 0 filename = 'gb_parliament.csv' # setup logger logging = logger.logger_config(with_file=False) logging.info('\n\nProgram starts..') # setup db connections and clean from previous data appES = databases.ES() appNeo = databases.Neo() appNeo.cleanNeo4j() # appES.cleanES() with open(filename, 'r') as file: logging.info("Reading the file {a}".format(a=filename)) for row in csv.DictReader(file): person = Person(row['id'], row['name'], row['sort_name'], row['email'], 'GB') # creating objects organization = Organization(row['group_id'], row['group']) membership = Membership(row['id'], row['group_id']) appNeo.insertPersonNeo4j(person) appNeo.insertOrganizationNeo4j(organization) appNeo.insertMembershipNeo4j(membership) appES.insertPersonES(person) appES.insertOrganizationES(organization) appES.insertMembershipES(membership) counter += 1 # close connections appES.close() appNeo.close() logging.info(f"Records read from a file: {counter}")
logger.info('Updating citations and references') min_days_to_update = age_days * 86400 papers = list(db_papers.find()) logger.info(f'Fetching {len(papers)} documents') for idx, p in enumerate(papers): if idx % 500 == 0: logger.info(f'Updating batch {idx}') cur_sem_sch = sem_sch_papers.find_one({'_id': p['_id']}) if not cur_sem_sch or (datetime.utcnow() - cur_sem_sch['last_rec_update'] ).total_seconds() > min_days_to_update: res = fetch_paper_data(p) if res: sem_sch_papers.update({'_id': res['_id']}, {'$set': res}, True) for a in res['authors']: sem_sch_authors.update({'_id': a['name']}, {}, True) else: logger.error('Failed to fetch paper data - {}'.format( p['_id'])) sleep(5) else: logger.debug('Paper is already in DB') logger.info('Finished updating refs') if __name__ == '__main__': from logger import logger_config logger_config(info_filename='citations_fetcher.log') update_all_papers()
import sys import threading import json import os import numpy as np import tensorflow as tf from flask import Flask, request, redirect, jsonify, render_template, url_for from foodnonfood import read_tensor_from_image_file from logger import logger_config UPLOAD_DIR = 'static/uploaded_images' # Directory to upload image files EXTENSIONS = ['.jpeg', '.jpg', '.png'] # file extensions which can use for inference LOGGER = logger_config('tfs_cli') tf.app.flags.DEFINE_string('version', None, 'spacify model version') tf.app.flags.DEFINE_boolean('debug', 'True', 'debug option') tf.app.flags.DEFINE_string('hostport', 'localhost:8501', 'host:port of predict server') FLAGS = tf.app.flags.FLAGS app = Flask(__name__) app.config['UPLOAD_DIR'] = UPLOAD_DIR def _predict(hostport, image_file): host, port = hostport.split(':') with tf.Session() as sess: image = read_tensor_from_image_file(sess, image_file)
import logging import schedule import time from fetch_papers import fetch_papers_main from twitter_daemon import main_twitter_fetcher from fetch_citations_and_references import update_all_papers from logger import logger_config if __name__ == '__main__': logger_config(info_filename='background_tasks.log') logger = logging.getLogger(__name__) logger.info('Start background tasks') schedule.every(30).minutes.do(main_twitter_fetcher) schedule.every(2).hours.do(fetch_papers_main) schedule.every().saturday.at("00:10").do(update_all_papers) while True: schedule.run_pending() time.sleep(1)
}).sort("time_published", pymongo.DESCENDING).limit(5)) papers = [{ 'name': p['title'], 'type': 'paper', 'authors': p['authors'] } for p in papers] return jsonify(authors + papers) # ----------------------------------------------------------------------------- # int main # ----------------------------------------------------------------------------- if __name__ == "__main__": logger_config() logger = logging.getLogger(__name__) parser = argparse.ArgumentParser() parser.add_argument('-p', '--prod', dest='prod', action='store_true', help='run in prod?') parser.add_argument('-r', '--num_results', dest='num_results', type=int, default=200, help='number of results to return per query') parser.add_argument('--port',
@catch_exceptions(logger=logger) def main_twitter_fetcher(): tweets = fetch_tweets() papers_to_update = process_tweets(tweets) summarize_tweets(papers_to_update) @catch_exceptions(logger=logger) def recalculate(): all_papers = list(db_papers.find({}, {'_id': 1})) summarize_tweets([p['_id'] for p in all_papers]) # ----------------------------------------------------------------------------- # authenticate to twitter API api = get_api_connector() # connect to mongodb instance client = pymongo.MongoClient() mdb = client.arxiv db_tweets = mdb.tweets # the "tweets" collection in "arxiv" database db_papers = mdb.papers # main loop if __name__ == '__main__': from logger import logger_config logger_config(info_filename='twitter_daemon.log') main_twitter_fetcher()
'extraction_conf': row.get('high_conf', '') == 'True', 'conferences': row.get('proceeding', ''), 'stars': int(row.get('stars', 0)), 'framework': row.get('framework'), 'datasets': row.get('datasets', '').split('|'), 'tasks': row.get('tasks', '').split('|'), 'paperswithcode_link': row.get('url', '') } try: papers.update(cur_id, {'$set': {'code': obj}}) except Exception as e: logger.error('Failed to update paper {} - {}'.format( cur_id['_id'], e)) else: logger.info('Paper not found - {}'.format(cur_id['_id'])) @catch_exceptions(logger=logger) def fetch_code_data(): logger.info('Fetching data from papers with code') data = fetch_data() logger.info('Updating DB with data from papers with code') update_db(data) logger.info('Finished updating data from papers with code') if __name__ == "__main__": from logger import logger_config logger_config(info_filename='arxiv_fetcher.log') fetch_code_data()
CommandHandler('cancel', cancel), MessageHandler(Filters.regex(ButtonPattern.CANCEL.value), cancel), MessageHandler(Filters.text, change_topic) ] }, fallbacks=[ CommandHandler('cancel', cancel), MessageHandler(Filters.regex(ButtonPattern.CANCEL.value), cancel), ]) updater.dispatcher.add_handler(CommandHandler('hello', hello)) updater.dispatcher.add_handler(CommandHandler('start', hello)) updater.dispatcher.add_handler(CommandHandler('current_topic', current_topic)) updater.dispatcher.add_handler( MessageHandler(Filters.regex(ButtonPattern.HELLO.value), hello)) updater.dispatcher.add_handler( MessageHandler(Filters.regex(ButtonPattern.TOPIC.value), current_topic)) updater.dispatcher.add_handler(conv_handler) updater.dispatcher.add_handler(MessageHandler(Filters.text, get_bot_response)) if __name__ == '__main__': _download_nltk_resources() logger.logger_config(level='DEBUG', root_level='WARNING') log.info('Bot has been started... \n Press ctrl + C to STOP the bot') updater.start_polling() updater.idle() log.info('Bot has been started')
``` """ import os import sys from flask import Flask, request, redirect, jsonify, render_template, url_for import tensorflow as tf from foodnonfood import FoodNonfood from logger import logger_config UPLOAD_DIR = 'static/uploaded_images' # Directory to upload image files EXTENSIONS = ['.jpeg', '.jpg', '.png'] # file extensions which can use for inference FOODNONFOOD = FoodNonfood() # Initialization of FoodNonfood classifier LOGGER = logger_config('flask_app') tf.app.flags.DEFINE_boolean('debug', 'True', 'debug option') tf.app.flags.DEFINE_string('model_path', None, 'file path of custom model') FLAGS = tf.app.flags.FLAGS app = Flask(__name__) app.config['UPLOAD_DIR'] = UPLOAD_DIR def _make_upload_dir(): if not os.path.isdir(UPLOAD_DIR): os.makedirs(UPLOAD_DIR) def _check_extension(filename):
if body['entity'] == 'organization': neo.removeRecordNeo4j('organization', body['group_id']) if body['entity'] == 'membership': neo.removeRecordNeo4j('membership', body['id']) neo.close() return "Deleted successfully" except Exception as e: logging.error(f'Error while deleting a record. Details: {e}', exc_info=True) neo.close() return f"Fail {e}" # Delete by id or group_id depending on the collection @app.route('/parliament_records_es', methods=['DELETE']) def delete_es(): try: es = ES() body = request.json print(body['entity']) es.removeRecordES(index='person', doc=body["queries"]) es.close() except Exception as e: logging.error(f'Error while deleting a record. Details: {e}', exc_info=True) es.close() return f"Fail {e}" if __name__ == "__main__": logging = logger.logger_config(with_file=True) config = load_configs() app.run(debug=False, host=config['kazdream']['url'], port=config['kazdream']['port'])