def main(path: str): _configs = load_configs(path) _batch_size = _configs['hyperparams']['batch_size'] _epochs = _configs['hyperparams']['epochs'] _embedding_size = _configs['hyperparams']['embedding_size'] _conv_dim = _configs['hyperparams']['convolution_dims'] _eval_steps = _configs['hyperparams']['evaluation_steps'] print('Getting dataset...') _data_manager = prepare_dataset(_configs) _vocab_size = _data_manager.get_vocabulary_size() _pretrained_embeddings = _data_manager.get_pretrained_word_embedding() print('Now build model...') model = build_model(_vocab_size, _embedding_size, _conv_dim, embeddings=_pretrained_embeddings) train_dataloader = _data_manager.get_train_data_loader(_batch_size) test_dataloader = _data_manager.get_test_data_loader(_batch_size) print('Train start!') train_manager = TrainManager(train_dataloader, test_dataloader, model, _epochs, _eval_steps) train_manager.train()
class TestServer(unittest.TestCase): CONFIGS = load_configs() def test_presence(self): test = create_presence_message('Guest', CONFIGS=self.CONFIGS) test[self.CONFIGS['TIME']] = 1.1 self.assertEqual( test, { self.CONFIGS['ACTION']: self.CONFIGS['PRESENCE'], self.CONFIGS['TIME']: 1.1, self.CONFIGS['USER']: { self.CONFIGS['ACCOUNT_NAME']: 'Guest' }, }) def test_correct_answer(self): self.assertEqual( handle_response({self.CONFIGS['RESPONSE']: 200}, self.CONFIGS), '200: OK') def test_bad_request(self): self.assertEqual( handle_response( { self.CONFIGS['RESPONSE']: 400, self.CONFIGS['ERROR']: 'Bad Request' }, self.CONFIGS), '400 : Bad Request') def test_no_response(self): self.assertRaises(ValueError, handle_response, {self.CONFIGS['ERROR']: 'Bad Request'}, self.CONFIGS)
def main(): global CONFIGS CONFIGS = load_configs(is_server=False) try: server_address = sys.argv[1] server_port = int(sys.argv[2]) if not 65535 >= server_port >= 1024: raise ValueError except IndexError: server_address = CONFIGS.get('DEFAULT_IP_ADDRESS') server_port = CONFIGS.get('DEFAULT_PORT') except ValueError: CLIENT_LOGGER.critical( f'The port should be specified in the range of (1024, 65535)') print('The port should be in the range of 1024 and 65535') sys.exit(1) transport = socket.socket(socket.AF_INET, socket.SOCK_STREAM) transport.connect((server_address, server_port)) presence_message = create_presence_message('Guest', CONFIGS) CLIENT_LOGGER.info(f'Sending a message to the server') send_message(transport, presence_message, CONFIGS) try: response = get_message(transport, CONFIGS) handled_response = handle_response(response, CONFIGS) CLIENT_LOGGER.debug(f'The response were handled: {response}') CLIENT_LOGGER.info(f'The response were handled: {handled_response}') print(f'The response from the server: {response}') print(handled_response) except (ValueError, json.JSONDecodeError): CLIENT_LOGGER.critical(f'An error during decoding of server message') print('Error message decoding ')
class Tests(unittest.TestCase): CONFIGS = load_configs(True) test_message_send = { CONFIGS['ACTION']: CONFIGS['PRESENCE'], CONFIGS['TIME']: 111111.111111, CONFIGS['USER']: { CONFIGS['ACCOUNT_NAME']: 'test_user' }, } test_success_receive = {CONFIGS['RESPONSE']: 200} test_error_receive = { CONFIGS['RESPONSE']: 400, CONFIGS['ERROR']: 'Bad Request', } def test_send_message(self): test_socket = TestSocket(self.test_message_send) send_message(test_socket, self.test_message_send, self.CONFIGS) self.assertEqual(test_socket.encoded_message, test_socket.received_message) with self.assertRaises(Exception): send_message(test_socket, test_socket, self.CONFIGS) def test_get_message(self): test_sock_ok = TestSocket(self.test_success_receive) test_sock_err = TestSocket(self.test_error_receive) self.assertEqual(get_message(test_sock_ok, self.CONFIGS), self.test_success_receive) self.assertEqual(get_message(test_sock_err, self.CONFIGS), self.test_error_receive)
def main(): global CONFIGS CONFIGS = load_configs(is_server=False) try: server_address = sys.argv[1] server_port = int(sys.argv[2]) if not 65535 >= server_port >= 1024: raise ValueError except IndexError: server_address = CONFIGS.get('DEFAULT_IP_ADDRESS') server_port = CONFIGS.get('DEFAULT_PORT') except ValueError: print('The port should be in the range of 1024 and 65535') sys.exit(1) transport = socket.socket(socket.AF_INET, socket.SOCK_STREAM) transport.connect((server_address, server_port)) presence_message = create_presence_message('Guest') send_message(transport, presence_message, CONFIGS) try: response = get_message(transport, CONFIGS) handled_response = handle_response(response) print(f'The response from the server: {response}') print(handled_response) except (ValueError, json.JSONDecodeError): print('Error message decoding ')
def __init__(self, options): self.options = options self.version = __init__.__version__ plugin_folders = [os.path.join(sys.path[0], 'plugins')] if os.path.exists('site.ini'): template.base_dir = os.path.join(options.dir, 'theme') self.verbose_log('Loading configuration files') self.configs = utils.load_configs(options.dir) config = utils.find_config(self.configs, '/') if config is not None: plugin_path = config.get('control', 'plugins_path') if plugin_path is not None: plugin_folders.append(os.path.join(os.getcwd(), plugin_path)) self.verbose_log('Initialising plugin manager') self.pm = PluginManager(plugin_folders) self.verbose_log(' - complete') if os.path.exists('site.ini'): self.verbose_log('Loading pages') self.pages = load_pages(options.dir, options.output, self) self.verbose_log(' - complete') self.commands = {} self.verbose_log('Running commands filter') apply_filter('commands', self.commands)
def main(path: str): _configs = load_configs(path) _batch_size = _configs['hyperparams']['batch_size'] _epochs = _configs['hyperparams']['epochs'] _eval_steps = _configs['hyperparams']['evaluation_steps'] _model_params = _configs['hyperparams']['model'] print('Getting dataset...') _data_manager = prepare_dataset(_configs) _vocab_size = _data_manager.get_vocabulary_size() _model_params['vocab_size'] = _vocab_size print('Now build model...') model = build_model(_model_params) train_dataloader = _data_manager.get_train_data_loader(_batch_size) test_dataloader = _data_manager.get_test_data_loader(100) print(model) print('Train start!') train_manager = TrainManager(train_dataloader, test_dataloader, model, _epochs, _eval_steps) train_manager.train()
class TestServer(unittest.TestCase): CONFIGS = load_configs(True) error_message = {CONFIGS['RESPONSE']: 400, CONFIGS['ERROR']: 'Bad Request'} success_message = {CONFIGS['RESPONSE']: 200} def test_without_action(self): self.assertEqual( handle_message( { self.CONFIGS['TIME']: '1.1', self.CONFIGS['USER']: { self.CONFIGS['ACCOUNT_NAME']: 'Guest' } }, self.CONFIGS), self.error_message) def test_wrong_action(self): self.assertEqual( handle_message( { self.CONFIGS['ACTION']: 'Wrong', self.CONFIGS['TIME']: '1.1', self.CONFIGS['USER']: { self.CONFIGS['ACCOUNT_NAME']: 'Guest' }, }, self.CONFIGS), self.error_message) def test_without_time(self): self.assertEqual( handle_message( { self.CONFIGS['ACTION']: self.CONFIGS['PRESENCE'], self.CONFIGS['USER']: { self.CONFIGS['ACCOUNT_NAME']: 'Guest' }, }, self.CONFIGS), self.error_message) def test_without_user(self): self.assertEqual( handle_message( { self.CONFIGS['ACTION']: self.CONFIGS['PRESENCE'], self.CONFIGS['TIME']: '1.1', }, self.CONFIGS), self.error_message) def test_wrong_user(self): self.assertEqual( handle_message( { self.CONFIGS['ACTION']: self.CONFIGS['PRESENCE'], self.CONFIGS['TIME']: 1.1, self.CONFIGS['USER']: { self.CONFIGS['ACCOUNT_NAME']: 'Guest1' } }, self.CONFIGS), self.error_message)
def main(): global CONFIGS logger.debug(f'The following configs are set \n. {CONFIGS}') CONFIGS = load_configs(is_server=False) logger.debug(f'For the key "is_server" set to "False" value') try: server_address = sys.argv[1] logger.debug(f'The server_address set to {server_address}') server_port = int(sys.argv[2]) logger.debug(f'The server_port set to {server_port}') if not 65535 >= server_port >= 1024: logger.error( f'The specified server_port out of range (1024, 65535). \n Entered value is {server_port}' f'\n Will be raised {ValueError}') raise ValueError except IndexError: server_address = CONFIGS.get('DEFAULT_IP_ADDRESS') server_port = CONFIGS.get('DEFAULT_PORT') logger.warning( f'A server_address and a server_port are not specified, will be set to default values \n' f' The server_address - {server_address} \n' f' The server_port {server_port}') except ValueError: logger.error( f'The specified server_port out of range (1024, 65535). \n Entered value is {server_port}' f'\n Will be raised {ValueError}') print('The port should be in the range of 1024 and 65535') sys.exit(1) transport = socket.socket(socket.AF_INET, socket.SOCK_STREAM) logger.debug(f'The socket was established for connection') transport.connect((server_address, server_port)) logger.debug( f'The connection with a server were established using {server_address} and {server_port}' ) presence_message = create_presence_message('Guest', CONFIGS) logger.info(f'The presence_message were created: \n' f'{presence_message}') send_message(transport, presence_message, CONFIGS) logger.debug( f'The presence_message were sent to the server with parameters: \n' f'socket - {transport}' f'presence message - {presence_message}' f'CONFIGS - {CONFIGS}') try: response = get_message(transport, CONFIGS) logger.debug(f'The response were received: \n {response}') handled_response = handle_response(response, CONFIGS) logger.info(f'The response were handled: \n {handled_response}') print(f'The response from the server: {response}') print(handled_response) except (ValueError, json.JSONDecodeError): logger.error( f'An error during decoding of server\'s response occurred. \n' f'The response - {response} \n' f'Error message - {ValueError}') print('Error message decoding ')
def __init__(self): """Called when the the GameWindow object is initialized. Initializes pygame and sets up our pygame window and other pygame tools.""" utils.load_configs() LOGGER.debug('Initializing window') # load and set up pygame pygame.init() # detect display resolution and shrink resolution if needed info = pygame.display.Info() displ_w, displ_h = info.current_w, info.current_h wnd_w, wnd_h = utils.SCREEN_SIZE if displ_w < wnd_w or displ_h < wnd_h: wnd_w, wnd_h = displ_w - 50, displ_h - 50 utils.SCREEN_SIZE = wnd_w, wnd_h # initial window position at the center of the screen center_x, center_y = (displ_w - wnd_w) / 2, (displ_h - wnd_h) / 2 os.environ['SDL_VIDEO_WINDOW_POS'] = str(center_x) + ',' + str(center_y) # create our window self.window = pygame.display.set_mode(utils.SCREEN_SIZE) # clock for ticking self.clock = pygame.time.Clock() # set the window title pygame.display.set_caption("Ancient Empires") # disable mouse pygame.mouse.set_visible(1) # tell pygame to only pay attention to certain events # we want to know if the user hits the X on the window, and we # want keys so we can close the window with the esc key #pygame.event.set_allowed([]) self._game_state = GameStateManager()
class TestSocket: CONFIGS = load_configs(True) def __init__(self, test_message): self.test_message = test_message self.encoded_message = None self.received_message = None def send(self, message_to_send): json_test_message = json.dumps(self.test_message) self.encoded_message = json_test_message.encode(self.CONFIGS['ENCODING']) self.received_message = message_to_send def recv(self, max_len): json_test_message = json.dumps(self.test_message) return json_test_message.encode(self.CONFIGS['ENCODING'])
def main(): global CONFIGS CONFIGS = load_configs() try: if '-p' in sys.argv: listen_port = int(sys.argv[sys.argv.index('-p') + 1]) else: listen_port = CONFIGS.get('DEFAULT_PORT') if not 65535 >= listen_port >= 1024: raise ValueError except IndexError: print('After -\'p\' should be specified the port') sys.exit(1) except ValueError: print('The port should be specified in the range from 1024 to 65535') sys.exit(1) try: if '-a' in sys.argv: listen_address = sys.argv[sys.argv.index('-a') + 1] else: listen_address = '' except IndexError: print('After -\'a\' - requires specify address for ') sys.exit(1) transport = socket.socket(socket.AF_INET, socket.SOCK_STREAM) transport.bind((listen_address, listen_port)) transport.listen(CONFIGS.get('MAX_CONNECTIONS')) while True: client, client_address = transport.accept() try: message = get_message(client, CONFIGS) response = handle_message(message) send_message(client, response, CONFIGS) # client.close() except (ValueError, json.JSONDecodeError): print('Received inappropriate message from a client') finally: client.close()
from flask import Flask from flask import render_template, redirect, request, jsonify, url_for from search_engine import index_dataset, search_phrase, query_database from flask_paginate import Pagination, get_page_args import utils import datetime, time import json import codecs import requests import os __location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__))) configs = utils.load_configs(os.path.join(__location__, 'config/config.yml')) app = Flask(__name__) @app.route('/') def index(): return render_template('index.html', title='A Simple Search Engine for arxiv dataset') @app.route('/search') def serp(): query = request.args.get('q', '') field = request.args.get('field', 'abstract') page = request.args.get('page', '1') _size = request.args.get('size', '50') deps = utils.get_deps(configs) page = int(page) _size = int(_size) print(query)
x = self.classifier(x) return x def get_model_state(self): L = [] for param_tensor in self.state_dict(): L.append('%s\t%s' % (param_tensor, self.state_dict()[param_tensor].size())) return L def get_optimizer_state(self): L = ['learning rate: %f' % self.optimizer.state_dict()['param_groups'][0]['lr'], 'momentum: %f' % self.optimizer.state_dict()['param_groups'][0]['momentum'], 'weight decay: %f' % self.optimizer.state_dict()['param_groups'][0]['weight_decay']] return L def _make_back_bone(cfg): return BackBone(cfg) def _make_classifier(cfg): # 新建全连接层 return Classifier(cfg) if __name__ == '__main__': from utils import load_configs cfg = load_configs('./config/vgg11.yaml') net = ImageClassifier(cfg) print(net)
def main(): global CONFIGS logger.debug(f'The following configs are set \n. {CONFIGS}') CONFIGS = load_configs() try: if '-p' in sys.argv: listen_port = int(sys.argv[sys.argv.index('-p') + 1]) logger.debug(f'The following listen_port is set - "{listen_port}"') else: listen_port = CONFIGS.get('DEFAULT_PORT') logger.debug( f'The DEFAULT_PORT listen_port is set - "{CONFIGS.get("DEFAULT_PORT")}"' ) if not 65535 >= listen_port >= 1024: logger.error( f'An error occurred on the server side during establishing listen_port. \n' f'The listen_port is out of range (1024, 65535)') raise ValueError except IndexError: logger.error( f'An IndexError error occurred on the server side, because the port was not specified' ) print('After -\'p\' should be specified the port') sys.exit(1) except ValueError: logger.error( f'An ValueError error occurred on the server side, because the port was ' f'specified out of range (1024, 65535)') print('The port should be specified in the range from 1024 to 65535') sys.exit(1) try: if '-a' in sys.argv: listen_address = sys.argv[sys.argv.index('-a') + 1] else: listen_address = '' except IndexError: print('After -\'a\' - requires specify address for ') sys.exit(1) transport = socket.socket(socket.AF_INET, socket.SOCK_STREAM) logger.debug(f'The socket were created on the server\'s side. \n' f'transport - {str(transport)}') transport.bind((listen_address, listen_port)) logger.debug(f'The binding of socket with the server\'s parameters. \n' f'listen_address - {listen_address}. \n' f'listen_port - {listen_port}. \n') transport.listen(CONFIGS.get('MAX_CONNECTIONS')) logger.debug(f'The number of connections established by server. \n' f'MAX_CONNECTIONS - {(CONFIGS.get("MAX_CONNECTIONS"))}') while True: client, client_address = transport.accept() logger.debug( f'Establishing connection with a client using the following: \n' f'client - {client}' f'client_address - {client_address}') try: message = get_message(client, CONFIGS) logger.debug(f'The message is received from the client: \n' f'message - {message}') response = handle_message(message, CONFIGS) logger.debug(f'The response is sent to the client: \n' f'response - {response}') send_message(client, response, CONFIGS) logger.debug(f'The message is sent: \n' f'client - {client} \n' f'response - {response}\n' f'CONFIGS - {CONFIGS}\n') # client.close() except (ValueError, json.JSONDecodeError): print('Received inappropriate message from a client') finally: client.close()
if keywords == None: keywords = cf["search_words"] t = Twarc(**cf['account']) # no keyword restriction but from a specific city # reason see this https://stackoverflow.com/questions/22889122/how-to-add-a-location-filter-to-tweepy-module if not os.path.isdir(city+"/"): os.makedirs(city) path = city + "/" + str(datetime.date.today())+".jsonl" locations = ",".join([str(i) for i in bbox[city]]) for tweet in t.filter(locations=locations): print("get one tweet") #TODO send_to_db(tweet) if __name__ == "__main__": cfs = load_configs() jobs = [] for i in range(len(cfs)): boxes = ["great_syd", "great_mel", "great_brisbane", "great_ald"] p = Process(target=stream_city, args=((cfs[i], boxes[i],)), daemon=True) jobs.append(p) p.start() [p.join() for p in jobs]
import sys import os import logging import logging.handlers from utils import load_configs CONFIGS = load_configs() sys.path.append('../') SERVER_FORMATTER = logging.Formatter('%(asctime)s %(levelname)s %(filename)s %(message)s') PATH = os.path.dirname(os.path.abspath(__file__)) PATH = os.path.join(PATH, 'server.log') STREAM_HANDLER = logging.StreamHandler(sys.stderr) STREAM_HANDLER.setFormatter(SERVER_FORMATTER) STREAM_HANDLER.setLevel(logging.ERROR) LOG_FILE = logging.handlers.TimedRotatingFileHandler(PATH, encoding='utf8', interval=1, when='D') LOG_FILE.setFormatter(SERVER_FORMATTER) LOGGER = logging.getLogger('server') LOGGER.addHandler(STREAM_HANDLER) LOGGER.addHandler(LOG_FILE) LOGGER.setLevel(CONFIGS.get('LOGGING_LEVEL', logging.DEBUG)) if __name__ == '__main__': LOGGER.critical('Critical error') LOGGER.error('error') LOGGER.debug('debug info')
from utils import tasks import utils import config import time from multiprocessing import Process #from raven.contrib.flask import Sentry #sentry = Sentry(config.SENTRY_DSN) app = config.create_app() mail = Mail(app) redis = redis.init_redis(app) # add exts for blueprint use app.config['redis'] = redis app.config['mail'] = mail app.config['configs'] = utils.load_configs() # ----- context processors # ----- end context processors # ----- template filters # ----- end filters @app.route('/') def index(): data = { 'version': getattr(config, 'APP_VERSION'), } return generate_api_response(data) @app.route('/notify', methods=['POST'])