def main(path: str):
_configs = load_configs(path)
_batch_size = _configs['hyperparams']['batch_size']
_epochs = _configs['hyperparams']['epochs']
_embedding_size = _configs['hyperparams']['embedding_size']
_conv_dim = _configs['hyperparams']['convolution_dims']
_eval_steps = _configs['hyperparams']['evaluation_steps']
print('Getting dataset...')
_data_manager = prepare_dataset(_configs)
_vocab_size = _data_manager.get_vocabulary_size()
_pretrained_embeddings = _data_manager.get_pretrained_word_embedding()
print('Now build model...')
model = build_model(_vocab_size, _embedding_size, _conv_dim, embeddings=_pretrained_embeddings)
train_dataloader = _data_manager.get_train_data_loader(_batch_size)
test_dataloader = _data_manager.get_test_data_loader(_batch_size)
print('Train start!')
train_manager = TrainManager(train_dataloader,
test_dataloader,
model,
_epochs,
_eval_steps)
train_manager.train()
class TestServer(unittest.TestCase):
CONFIGS = load_configs()
def test_presence(self):
test = create_presence_message('Guest', CONFIGS=self.CONFIGS)
test[self.CONFIGS['TIME']] = 1.1
self.assertEqual(
test, {
self.CONFIGS['ACTION']: self.CONFIGS['PRESENCE'],
self.CONFIGS['TIME']: 1.1,
self.CONFIGS['USER']: {
self.CONFIGS['ACCOUNT_NAME']: 'Guest'
},
})
def test_correct_answer(self):
self.assertEqual(
handle_response({self.CONFIGS['RESPONSE']: 200}, self.CONFIGS),
'200: OK')
def test_bad_request(self):
self.assertEqual(
handle_response(
{
self.CONFIGS['RESPONSE']: 400,
self.CONFIGS['ERROR']: 'Bad Request'
}, self.CONFIGS), '400 : Bad Request')
def test_no_response(self):
self.assertRaises(ValueError, handle_response,
{self.CONFIGS['ERROR']: 'Bad Request'}, self.CONFIGS)
def main():
global CONFIGS
CONFIGS = load_configs(is_server=False)
try:
server_address = sys.argv[1]
server_port = int(sys.argv[2])
if not 65535 >= server_port >= 1024:
raise ValueError
except IndexError:
server_address = CONFIGS.get('DEFAULT_IP_ADDRESS')
server_port = CONFIGS.get('DEFAULT_PORT')
except ValueError:
CLIENT_LOGGER.critical(
f'The port should be specified in the range of (1024, 65535)')
print('The port should be in the range of 1024 and 65535')
sys.exit(1)
transport = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
transport.connect((server_address, server_port))
presence_message = create_presence_message('Guest', CONFIGS)
CLIENT_LOGGER.info(f'Sending a message to the server')
send_message(transport, presence_message, CONFIGS)
try:
response = get_message(transport, CONFIGS)
handled_response = handle_response(response, CONFIGS)
CLIENT_LOGGER.debug(f'The response were handled: {response}')
CLIENT_LOGGER.info(f'The response were handled: {handled_response}')
print(f'The response from the server: {response}')
print(handled_response)
except (ValueError, json.JSONDecodeError):
CLIENT_LOGGER.critical(f'An error during decoding of server message')
print('Error message decoding ')
class Tests(unittest.TestCase):
CONFIGS = load_configs(True)
test_message_send = {
CONFIGS['ACTION']: CONFIGS['PRESENCE'],
CONFIGS['TIME']: 111111.111111,
CONFIGS['USER']: {
CONFIGS['ACCOUNT_NAME']: 'test_user'
},
}
test_success_receive = {CONFIGS['RESPONSE']: 200}
test_error_receive = {
CONFIGS['RESPONSE']: 400,
CONFIGS['ERROR']: 'Bad Request',
}
def test_send_message(self):
test_socket = TestSocket(self.test_message_send)
send_message(test_socket, self.test_message_send, self.CONFIGS)
self.assertEqual(test_socket.encoded_message,
test_socket.received_message)
with self.assertRaises(Exception):
send_message(test_socket, test_socket, self.CONFIGS)
def test_get_message(self):
test_sock_ok = TestSocket(self.test_success_receive)
test_sock_err = TestSocket(self.test_error_receive)
self.assertEqual(get_message(test_sock_ok, self.CONFIGS),
self.test_success_receive)
self.assertEqual(get_message(test_sock_err, self.CONFIGS),
self.test_error_receive)
def main():
global CONFIGS
CONFIGS = load_configs(is_server=False)
try:
server_address = sys.argv[1]
server_port = int(sys.argv[2])
if not 65535 >= server_port >= 1024:
raise ValueError
except IndexError:
server_address = CONFIGS.get('DEFAULT_IP_ADDRESS')
server_port = CONFIGS.get('DEFAULT_PORT')
except ValueError:
print('The port should be in the range of 1024 and 65535')
sys.exit(1)
transport = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
transport.connect((server_address, server_port))
presence_message = create_presence_message('Guest')
send_message(transport, presence_message, CONFIGS)
try:
response = get_message(transport, CONFIGS)
handled_response = handle_response(response)
print(f'The response from the server: {response}')
print(handled_response)
except (ValueError, json.JSONDecodeError):
print('Error message decoding ')
def __init__(self, options):
self.options = options
self.version = __init__.__version__
plugin_folders = [os.path.join(sys.path[0], 'plugins')]
if os.path.exists('site.ini'):
template.base_dir = os.path.join(options.dir, 'theme')
self.verbose_log('Loading configuration files')
self.configs = utils.load_configs(options.dir)
config = utils.find_config(self.configs, '/')
if config is not None:
plugin_path = config.get('control', 'plugins_path')
if plugin_path is not None:
plugin_folders.append(os.path.join(os.getcwd(), plugin_path))
self.verbose_log('Initialising plugin manager')
self.pm = PluginManager(plugin_folders)
self.verbose_log(' - complete')
if os.path.exists('site.ini'):
self.verbose_log('Loading pages')
self.pages = load_pages(options.dir, options.output, self)
self.verbose_log(' - complete')
self.commands = {}
self.verbose_log('Running commands filter')
apply_filter('commands', self.commands)
def main(path: str):
_configs = load_configs(path)
_batch_size = _configs['hyperparams']['batch_size']
_epochs = _configs['hyperparams']['epochs']
_eval_steps = _configs['hyperparams']['evaluation_steps']
_model_params = _configs['hyperparams']['model']
print('Getting dataset...')
_data_manager = prepare_dataset(_configs)
_vocab_size = _data_manager.get_vocabulary_size()
_model_params['vocab_size'] = _vocab_size
print('Now build model...')
model = build_model(_model_params)
train_dataloader = _data_manager.get_train_data_loader(_batch_size)
test_dataloader = _data_manager.get_test_data_loader(100)
print(model)
print('Train start!')
train_manager = TrainManager(train_dataloader, test_dataloader, model,
_epochs, _eval_steps)
train_manager.train()
class TestServer(unittest.TestCase):
CONFIGS = load_configs(True)
error_message = {CONFIGS['RESPONSE']: 400, CONFIGS['ERROR']: 'Bad Request'}
success_message = {CONFIGS['RESPONSE']: 200}
def test_without_action(self):
self.assertEqual(
handle_message(
{
self.CONFIGS['TIME']: '1.1',
self.CONFIGS['USER']: {
self.CONFIGS['ACCOUNT_NAME']: 'Guest'
}
}, self.CONFIGS), self.error_message)
def test_wrong_action(self):
self.assertEqual(
handle_message(
{
self.CONFIGS['ACTION']: 'Wrong',
self.CONFIGS['TIME']: '1.1',
self.CONFIGS['USER']: {
self.CONFIGS['ACCOUNT_NAME']: 'Guest'
},
}, self.CONFIGS), self.error_message)
def test_without_time(self):
self.assertEqual(
handle_message(
{
self.CONFIGS['ACTION']: self.CONFIGS['PRESENCE'],
self.CONFIGS['USER']: {
self.CONFIGS['ACCOUNT_NAME']: 'Guest'
},
}, self.CONFIGS), self.error_message)
def test_without_user(self):
self.assertEqual(
handle_message(
{
self.CONFIGS['ACTION']: self.CONFIGS['PRESENCE'],
self.CONFIGS['TIME']: '1.1',
}, self.CONFIGS), self.error_message)
def test_wrong_user(self):
self.assertEqual(
handle_message(
{
self.CONFIGS['ACTION']: self.CONFIGS['PRESENCE'],
self.CONFIGS['TIME']: 1.1,
self.CONFIGS['USER']: {
self.CONFIGS['ACCOUNT_NAME']: 'Guest1'
}
}, self.CONFIGS), self.error_message)
def main():
global CONFIGS
logger.debug(f'The following configs are set \n. {CONFIGS}')
CONFIGS = load_configs(is_server=False)
logger.debug(f'For the key "is_server" set to "False" value')
try:
server_address = sys.argv[1]
logger.debug(f'The server_address set to {server_address}')
server_port = int(sys.argv[2])
logger.debug(f'The server_port set to {server_port}')
if not 65535 >= server_port >= 1024:
logger.error(
f'The specified server_port out of range (1024, 65535). \n Entered value is {server_port}'
f'\n Will be raised {ValueError}')
raise ValueError
except IndexError:
server_address = CONFIGS.get('DEFAULT_IP_ADDRESS')
server_port = CONFIGS.get('DEFAULT_PORT')
logger.warning(
f'A server_address and a server_port are not specified, will be set to default values \n'
f' The server_address - {server_address} \n'
f' The server_port {server_port}')
except ValueError:
logger.error(
f'The specified server_port out of range (1024, 65535). \n Entered value is {server_port}'
f'\n Will be raised {ValueError}')
print('The port should be in the range of 1024 and 65535')
sys.exit(1)
transport = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
logger.debug(f'The socket was established for connection')
transport.connect((server_address, server_port))
logger.debug(
f'The connection with a server were established using {server_address} and {server_port}'
)
presence_message = create_presence_message('Guest', CONFIGS)
logger.info(f'The presence_message were created: \n' f'{presence_message}')
send_message(transport, presence_message, CONFIGS)
logger.debug(
f'The presence_message were sent to the server with parameters: \n'
f'socket - {transport}'
f'presence message - {presence_message}'
f'CONFIGS - {CONFIGS}')
try:
response = get_message(transport, CONFIGS)
logger.debug(f'The response were received: \n {response}')
handled_response = handle_response(response, CONFIGS)
logger.info(f'The response were handled: \n {handled_response}')
print(f'The response from the server: {response}')
print(handled_response)
except (ValueError, json.JSONDecodeError):
logger.error(
f'An error during decoding of server\'s response occurred. \n'
f'The response - {response} \n'
f'Error message - {ValueError}')
print('Error message decoding ')
def __init__(self):
"""Called when the the GameWindow object is initialized. Initializes
pygame and sets up our pygame window and other pygame tools."""
utils.load_configs()
LOGGER.debug('Initializing window')
# load and set up pygame
pygame.init()
# detect display resolution and shrink resolution if needed
info = pygame.display.Info()
displ_w, displ_h = info.current_w, info.current_h
wnd_w, wnd_h = utils.SCREEN_SIZE
if displ_w < wnd_w or displ_h < wnd_h:
wnd_w, wnd_h = displ_w - 50, displ_h - 50
utils.SCREEN_SIZE = wnd_w, wnd_h
# initial window position at the center of the screen
center_x, center_y = (displ_w - wnd_w) / 2, (displ_h - wnd_h) / 2
os.environ['SDL_VIDEO_WINDOW_POS'] = str(center_x) + ',' + str(center_y)
# create our window
self.window = pygame.display.set_mode(utils.SCREEN_SIZE)
# clock for ticking
self.clock = pygame.time.Clock()
# set the window title
pygame.display.set_caption("Ancient Empires")
# disable mouse
pygame.mouse.set_visible(1)
# tell pygame to only pay attention to certain events
# we want to know if the user hits the X on the window, and we
# want keys so we can close the window with the esc key
#pygame.event.set_allowed([])
self._game_state = GameStateManager()
class TestSocket:
CONFIGS = load_configs(True)
def __init__(self, test_message):
self.test_message = test_message
self.encoded_message = None
self.received_message = None
def send(self, message_to_send):
json_test_message = json.dumps(self.test_message)
self.encoded_message = json_test_message.encode(self.CONFIGS['ENCODING'])
self.received_message = message_to_send
def recv(self, max_len):
json_test_message = json.dumps(self.test_message)
return json_test_message.encode(self.CONFIGS['ENCODING'])
def main():
global CONFIGS
CONFIGS = load_configs()
try:
if '-p' in sys.argv:
listen_port = int(sys.argv[sys.argv.index('-p') + 1])
else:
listen_port = CONFIGS.get('DEFAULT_PORT')
if not 65535 >= listen_port >= 1024:
raise ValueError
except IndexError:
print('After -\'p\' should be specified the port')
sys.exit(1)
except ValueError:
print('The port should be specified in the range from 1024 to 65535')
sys.exit(1)
try:
if '-a' in sys.argv:
listen_address = sys.argv[sys.argv.index('-a') + 1]
else:
listen_address = ''
except IndexError:
print('After -\'a\' - requires specify address for ')
sys.exit(1)
transport = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
transport.bind((listen_address, listen_port))
transport.listen(CONFIGS.get('MAX_CONNECTIONS'))
while True:
client, client_address = transport.accept()
try:
message = get_message(client, CONFIGS)
response = handle_message(message)
send_message(client, response, CONFIGS)
# client.close()
except (ValueError, json.JSONDecodeError):
print('Received inappropriate message from a client')
finally:
client.close()
from flask import Flask
from flask import render_template, redirect, request, jsonify, url_for
from search_engine import index_dataset, search_phrase, query_database
from flask_paginate import Pagination, get_page_args
import utils
import datetime, time
import json
import codecs
import requests
import os
__location__ = os.path.realpath(os.path.join(os.getcwd(), os.path.dirname(__file__)))
configs = utils.load_configs(os.path.join(__location__, 'config/config.yml'))
app = Flask(__name__)
@app.route('/')
def index():
return render_template('index.html', title='A Simple Search Engine for arxiv dataset')
@app.route('/search')
def serp():
query = request.args.get('q', '')
field = request.args.get('field', 'abstract')
page = request.args.get('page', '1')
_size = request.args.get('size', '50')
deps = utils.get_deps(configs)
page = int(page)
_size = int(_size)
print(query)
x = self.classifier(x)
return x
def get_model_state(self):
L = []
for param_tensor in self.state_dict():
L.append('%s\t%s' % (param_tensor, self.state_dict()[param_tensor].size()))
return L
def get_optimizer_state(self):
L = ['learning rate: %f' % self.optimizer.state_dict()['param_groups'][0]['lr'],
'momentum: %f' % self.optimizer.state_dict()['param_groups'][0]['momentum'],
'weight decay: %f' % self.optimizer.state_dict()['param_groups'][0]['weight_decay']]
return L
def _make_back_bone(cfg):
return BackBone(cfg)
def _make_classifier(cfg):
# 新建全连接层
return Classifier(cfg)
if __name__ == '__main__':
from utils import load_configs
cfg = load_configs('./config/vgg11.yaml')
net = ImageClassifier(cfg)
print(net)
def main():
global CONFIGS
logger.debug(f'The following configs are set \n. {CONFIGS}')
CONFIGS = load_configs()
try:
if '-p' in sys.argv:
listen_port = int(sys.argv[sys.argv.index('-p') + 1])
logger.debug(f'The following listen_port is set - "{listen_port}"')
else:
listen_port = CONFIGS.get('DEFAULT_PORT')
logger.debug(
f'The DEFAULT_PORT listen_port is set - "{CONFIGS.get("DEFAULT_PORT")}"'
)
if not 65535 >= listen_port >= 1024:
logger.error(
f'An error occurred on the server side during establishing listen_port. \n'
f'The listen_port is out of range (1024, 65535)')
raise ValueError
except IndexError:
logger.error(
f'An IndexError error occurred on the server side, because the port was not specified'
)
print('After -\'p\' should be specified the port')
sys.exit(1)
except ValueError:
logger.error(
f'An ValueError error occurred on the server side, because the port was '
f'specified out of range (1024, 65535)')
print('The port should be specified in the range from 1024 to 65535')
sys.exit(1)
try:
if '-a' in sys.argv:
listen_address = sys.argv[sys.argv.index('-a') + 1]
else:
listen_address = ''
except IndexError:
print('After -\'a\' - requires specify address for ')
sys.exit(1)
transport = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
logger.debug(f'The socket were created on the server\'s side. \n'
f'transport - {str(transport)}')
transport.bind((listen_address, listen_port))
logger.debug(f'The binding of socket with the server\'s parameters. \n'
f'listen_address - {listen_address}. \n'
f'listen_port - {listen_port}. \n')
transport.listen(CONFIGS.get('MAX_CONNECTIONS'))
logger.debug(f'The number of connections established by server. \n'
f'MAX_CONNECTIONS - {(CONFIGS.get("MAX_CONNECTIONS"))}')
while True:
client, client_address = transport.accept()
logger.debug(
f'Establishing connection with a client using the following: \n'
f'client - {client}'
f'client_address - {client_address}')
try:
message = get_message(client, CONFIGS)
logger.debug(f'The message is received from the client: \n'
f'message - {message}')
response = handle_message(message, CONFIGS)
logger.debug(f'The response is sent to the client: \n'
f'response - {response}')
send_message(client, response, CONFIGS)
logger.debug(f'The message is sent: \n'
f'client - {client} \n'
f'response - {response}\n'
f'CONFIGS - {CONFIGS}\n')
# client.close()
except (ValueError, json.JSONDecodeError):
print('Received inappropriate message from a client')
finally:
client.close()
if keywords == None:
keywords = cf["search_words"]
t = Twarc(**cf['account'])
# no keyword restriction but from a specific city
# reason see this https://stackoverflow.com/questions/22889122/how-to-add-a-location-filter-to-tweepy-module
if not os.path.isdir(city+"/"):
os.makedirs(city)
path = city + "/" + str(datetime.date.today())+".jsonl"
locations = ",".join([str(i) for i in bbox[city]])
for tweet in t.filter(locations=locations):
print("get one tweet") #TODO
send_to_db(tweet)
if __name__ == "__main__":
cfs = load_configs()
jobs = []
for i in range(len(cfs)):
boxes = ["great_syd", "great_mel", "great_brisbane", "great_ald"]
p = Process(target=stream_city, args=((cfs[i], boxes[i],)), daemon=True)
jobs.append(p)
p.start()
[p.join() for p in jobs]
import sys
import os
import logging
import logging.handlers
from utils import load_configs
CONFIGS = load_configs()
sys.path.append('../')
SERVER_FORMATTER = logging.Formatter('%(asctime)s %(levelname)s %(filename)s %(message)s')
PATH = os.path.dirname(os.path.abspath(__file__))
PATH = os.path.join(PATH, 'server.log')
STREAM_HANDLER = logging.StreamHandler(sys.stderr)
STREAM_HANDLER.setFormatter(SERVER_FORMATTER)
STREAM_HANDLER.setLevel(logging.ERROR)
LOG_FILE = logging.handlers.TimedRotatingFileHandler(PATH, encoding='utf8', interval=1, when='D')
LOG_FILE.setFormatter(SERVER_FORMATTER)
LOGGER = logging.getLogger('server')
LOGGER.addHandler(STREAM_HANDLER)
LOGGER.addHandler(LOG_FILE)
LOGGER.setLevel(CONFIGS.get('LOGGING_LEVEL', logging.DEBUG))
if __name__ == '__main__':
LOGGER.critical('Critical error')
LOGGER.error('error')
LOGGER.debug('debug info')
from utils import tasks
import utils
import config
import time
from multiprocessing import Process
#from raven.contrib.flask import Sentry
#sentry = Sentry(config.SENTRY_DSN)
app = config.create_app()
mail = Mail(app)
redis = redis.init_redis(app)
# add exts for blueprint use
app.config['redis'] = redis
app.config['mail'] = mail
app.config['configs'] = utils.load_configs()
# ----- context processors
# ----- end context processors
# ----- template filters
# ----- end filters
@app.route('/')
def index():
data = {
'version': getattr(config, 'APP_VERSION'),
}
return generate_api_response(data)
@app.route('/notify', methods=['POST'])