def lookup(self, root):
# form a file list to distribute
log('Looking for files in {root}...'.format(root=root))
for s in os.listdir(root):
if '.' and self.delimiter in s:
self.file_dict[s] = ''
time.sleep(self.refresh_interval)
def __init__(self, config):
self.refresh_interval = config['distribution_service'][
'refresh_interval']
self.delimiter = config['distribution_service']['delimiter']
self.roots = config['distribution_service']['roots']
self.file_dict = {
} # this file list is a dict of a kind 'file name': 'generated destination'
log('Distribution service initialized.')
def __init__(self, json_config):
self.running = True
self.config = json_config
self.wait = json_config['wait']
self.xmlrpc_client = xmlrpc.client.ServerProxy(
'http://{host}:{port}/RPC2'.format(host=self.config['server_host'],
port=self.config['server_port']))
self.creds = (json_config['olymp_login'], json_config['olymp_password'])
log('JSON Updater initialized.')
def __init__(self, config):
super(JsonBlazer, self).__init__(config)
self.outcome_dict = {} # a dict of a kind 'П1': '2.22' for managing multiple bet types
self.json_data = None
self.outcomes = {} # outcomes is a list of dicts {'match_name': [{key: value}, {key: value}]}
self.sports = {} # sports is a dict of {'football': [{keys: values}, {keys: values}, {keys: values}]}
self.json_paths = config['server_data_files']
self.parse_json()
log('JSON blazer initialized')
def move_to_new_path(self, path, name, root):
# preparing to move
original_full_file_path = os.path.join(root, name)
new_file_path = os.path.join(root, path)
log('Checking dir {dir}...'.format(dir=new_file_path))
os.makedirs(new_file_path, exist_ok=True)
# moving here
log('Moving {name} to {path}...'.format(name=name, path=path))
shutil.move(original_full_file_path, os.path.join(new_file_path, name))
def return_team_names(self, topic_name, sport_type='Футбол'):
# check on empty
if len(self.sports) == 0:
log('JSON is empty.')
return False
# determine team names by topic_name
for i in self.sports[sport_type]:
if i['topic'] == topic_name:
return ','.join(i['teams'])
def return_logos(self, sport='Футбол'):
# check on empty
if len(self.sports) == 0:
log('JSON is empty.')
return False
# call Storage Ravager
logo1, logo2 = self.get_team_logos(team1=self.sports['Футбол'][0]['teams'][0],
team2=self.sports['Футбол'][0]['teams'][1])
log('Requesting logos by Storage Ravager for: {sport}'.format(sport=sport))
return ','.join([logo1, logo2])
def return_all_sports(self):
log('Sport types available:')
sport_names = []
for sport in self.sports:
sport_names.extend([*sport])
# show found sports
log([*sport][0])
# return comma separated string
return ','.join(sport_names)
def parse_single_outcome(self, outcome):
if outcome:
names, values = outcome.split('\n')
names_list = names.split(',')
values_list = values.split(',')
# values and names lists should be same length, else below method wont work
if len(names_list) == len(values_list):
for name in names_list:
self.outcome_dict[name] = values_list[names_list.index(name)]
else:
log('Length discrepancy: {names} -- {values}'.format(names=names_list,
values=values_list))
return False
else:
log('Unable to parse outcome because its empty.')
def parse_json(self):
# to avoid json_data stacking erase existing json_data
self.flush()
for path in self.json_paths:
self.json_data.extend(get_data_file_list(path))
# check if its empty, then parse
if len(self.json_data) > 0:
for i in self.json_data:
if i['sport_type_title'] in self.sports:
sub_dict = {'id': i['sport_type_id'],
'topic': i['topic'],
'event_id': str(i['event_id']),
'event_title': i['event_title'],
'teams': (i['team1'], i['team2'],),
'start_date2': i['start_date2'],
'start_date_timestamp': str(i['start_date_timestamp']),
'url': i['url'],
'url_mobile': i['url_mobile']
}
self.sports[i['sport_type_title']].append(sub_dict)
else:
self.sports.update(
{i['sport_type_title']: [
{'id': i['sport_type_id'],
'topic': i['topic'],
'event_id': str(i['event_id']),
'event_title': i['event_title'],
'teams': (i['team1'], i['team2'],),
'start_date2': i['start_date2'],
'start_date_timestamp': str(i['start_date_timestamp']),
'url': i['url'],
'url_mobile': i['url_mobile']
}
]
}
)
sub_dict_outcomes = {i['topic']: i['outcomes']}
# if we'll need to find specific outcomes do it here
self.outcomes.update(sub_dict_outcomes)
return True
else:
log('JSON file is empty.')
return False
def return_team_data(sub_path):
# should return available collections content (records)
schema, collection = sub_path.split('/', 1)
# get schema (database) access
db = client[schema]
# get a collection (table) access
teams = db[collection]
# search in collection
team_document = teams.find_one({"name_olimp": request.args['team']})
# handle empty result
try:
return json.dumps(team_document, cls=JSONEncoder)
except Exception as e:
log('An exception occurred: %s' % e)
return str(e)
def run(self):
log('Distribution service is running...')
while True:
for root in self.roots:
# update file list
self.lookup(root)
# generate paths for found files
for name in self.file_dict:
self.process_filename(name, root)
# move files and delete (not copy), log it
names_to_delete = []
for name, path in self.file_dict.items():
self.move_to_new_path(path, name, root)
names_to_delete.append(name)
# deleting processed file from files_dict
for name in names_to_delete:
log('Removing deleted file name from files dict: %s' %
self.file_dict.pop(name, None))
def return_outcomes(self, topic_name, bet_types):
# check bet_types
if not bet_types:
log('No bet types found for {topic}.'.format(topic=topic_name))
return False
# ensure that bet_types is actually a several comma-separated types of bets
if not ',' in bet_types:
if len(bet_types) > 0:
bet_types += ','
else:
log('Incorrect input: bet_types should be a csv json_data or a single string which is a bet type name.')
return False
# ensure that outcome_dict is empty
if self.outcome_dict:
self.outcome_dict = {}
# loop via bet types and get all outcomes from each bet type
for bet_type in bet_types.split(','):
if bet_type:
outcome = self.return_outcome(topic_name, bet_type)
self.parse_single_outcome(outcome)
# parse fresh formed outcomes dict to provide output string of a kind 'П1,П2,Х,П3,П4,Ч;1,2,0,3,4'
outcomes = self.prepare_outcomes_string()
log('All available bets for {bet_types}: {outcomes}'.format(bet_types=bet_types,
outcomes=outcomes))
return outcomes
def get_presenter(self, presenter):
# form request url
url = 'http://{host}:{port}/api/{database}/Teams'.format(host=self.config['storage_app']['host'],
port=self.config['storage_app']['port'],
database=self.config['storage_app']['database_name'])
payload = {'presenter': presenter}
r = requests.get(url, params=payload)
if r.status_code != 200:
log('Database returned response code %d.' % r.status_code)
log('Please check StorageApp log, collection or collection field may not exist.')
return r.status_code
# parse text
response_dict = json.loads(r.text)
# get presenter's name
presenter_name = response_dict['name']
presenter_position = response_dict['position']
presenter_social = response_dict['vkpage']
# returns string
log('Returning presenter info: {name}, position: {position}, social: {social}'.format(name=presenter_name,
position=presenter_position,
social=presenter_social))
return ','.join([presenter_name, presenter_position, presenter_social])
def get_match_start_date(self, topic_name, sport_type='Футбол'):
return_template = 'hh:mm'
# check on empty
if len(self.outcomes[topic_name]) == 0:
log('JSON is empty.')
return False
timestamp = None
# find start_date_timestamp by topic_name
for i in self.sports[sport_type]:
if i['topic'] == topic_name:
timestamp = i['start_date_timestamp']
break
# check if timestamp found - return it
# if not - return False
if timestamp:
return parse_13_digits_timestamp(timestamp, return_template)
else:
log("Unable to find start_date_timestamp: topic mismatch")
return False
def return_bet_types(self, topic_name):
# check on empty
if len(self.outcomes[topic_name]) == 0:
log('JSON is empty.')
return False
# return all bet types for topic_name 'main,nextgoal'
try:
bet_types = ','.join(self.outcomes[topic_name].keys())
log('Available bet types for {topic}: {bet_types}.'.format(topic=topic_name,
bet_types=bet_types))
return bet_types
except KeyError:
log('Unable to return bet types for %s.' % topic_name)
log('No topic %s.' % topic_name)
return False
def return_outcome(self, topic_name, bet_type):
# return values 'П1,Х,П2;1,2,3'
try:
bets = {}
# fill bets
for outcome in self.outcomes[topic_name][bet_type]:
bets[outcome['bet_title']] = str(outcome['factor_value'])
# now sort it as commented above
names = []
values = []
for key, value in bets.items():
names.append(key)
values.append(value)
bets_in_string = '{bet_titles}\n{values}'.format(bet_titles=','.join(names),
values=','.join(values))
log('Available bets for {bet_type}: {bets}'.format(bet_type=bet_type,
bets=bets_in_string))
return bets_in_string
except KeyError:
log('Unable to find outcome for {topic} and {bet_type}.'.format(topic=topic_name,
bet_type=bet_type))
return False
def return_all_topics(self, topic='Футбол'):
# check on empty
if len(self.sports[topic]) == 0:
log('JSON is empty.')
return False
# proceed if not empty
topics = []
log('Available events for {topic}:'.format(topic=topic))
for i in self.sports[topic]:
topics.append(i['topic'])
log(i['topic'])
return ','.join(topics)
def get_team_short_name_localized(self, team):
# form request url
url = 'http://{host}:{port}/api/{database}/Teams'.format(host=self.config['storage_app']['host'],
port=self.config['storage_app']['port'],
database=self.config['storage_app']['database_name'])
payload = {'team': team}
r = requests.get(url, params=payload)
if r.status_code != 200:
log('Database returned response code %d.' % r.status_code)
log('Please check StorageApp log, collection or collection field may not exist.')
return r.status_code
# parse text
response_dict = json.loads(r.text)
# get logo path
name_short_localized = response_dict['name_short_localized']
# returns string
log('Returning short name localized: %s' % name_short_localized)
return name_short_localized
def return_leagues(self, sport_type):
log('Leagues available for %s:' % sport_type)
for sport_dict in self.sports:
if sport_type in sport_dict.keys():
sport_id = sport_dict[sport_type]
break
try:
league_names = []
for league in self.league:
for league_name, league_info in league.items():
if sport_id == league_info[0]:
league_names.append(league_name)
# return found leagues
for l in league_names:
log(l)
# return comma separated string
return ','.join(league_names)
except NameError:
log('{sport_type} does not exist in current JSON version.'.format(sport_type=sport_type))
return '{sport_type} does not exist in current JSON version.'.format(sport_type=sport_type)
def __init__(self, config):
super(StorageRavager, self).__init__()
self.config = config
log('Storage Ravager initialized.')
@app.route('/')
@app.route('/login', methods=['GET', 'POST'])
def login():
if request.method == 'POST':
# get user/pass from form
session['username'] = request.form['username']
session['password'] = request.form['password']
# check user/pass pair
# TODO check user via StorageApp
# TODO check password via StorageApp for checked user
# if check are OK redirect to Main, else abort(403)
return redirect(url_for('main'))
return render_template('login.html')
@app.route('/logout')
def logout():
# remove the username from the session if it's there
session.pop('username', None)
return redirect(url_for('login'))
if __name__ == '__main__':
# start reaping
log('Starting to reap...')
app.run(debug=False)
teams = db[collection]
# search in collection
team_document = teams.find_one({"name_olimp": request.args['team']})
# handle empty result
try:
return json.dumps(team_document, cls=JSONEncoder)
except Exception as e:
log('An exception occurred: %s' % e)
return str(e)
if __name__ == '__main__':
# settings
# read config
json_config = json_reader('config.json')
MONGO_HOST = json_config['mongo']['host']
MONGO_PORT = json_config['mongo']['port']
storage_app_host = json_config['storage_app']['host']
storage_app_port = json_config['storage_app']['port']
# init client for 'media' database
client = MongoClient(MONGO_HOST, int(MONGO_PORT))
log('Created MongoDB client.')
# starting database manager
log('Starting database manager...')
app.run(debug=False, host=storage_app_host, port=storage_app_port)
try:
# prepare server log
prepare_logfile()
# read config
json_config = json_reader('config.json')
# Create server
host = json_config['server_host']
port = json_config['server_port']
server = SimpleXMLRPCServer((host, int(port)))
# Register methods
server.register_introspection_functions()
server.register_function(get_os, 'get_os')
# Register an instance; all the methods of the instance are published as XML-RPC methods
server.register_instance(DevourersEater(json_config))
# init json updater and put it in its own thread
ju = JsonUpdaterDaemon(json_config)
updater_thread = threading.Thread(target=ju.run)
updater_thread.start()
# Run the server's main loop
log('Server is running on {host}:{port}.'.format(host=host, port=port))
server.serve_forever()
except KeyboardInterrupt:
log('Interrupted from keyboard. Stopped.')
sys.exit(0)
def refresh_data(self):
self.xmlrpc_client.parse_json()
log('In-memory JSON refreshed.')
def update(self):
if self.config:
for source in self.config['data_source']:
log('Requesting json_data source {url}, to be written to "{file}"'.format(
url=self.config['data_source'][source]['url'],
file=self.config['data_source'][source]['file']))
try:
r = requests.get(self.config['data_source'][source]['url'], auth=self.creds)
if r.status_code == 200:
with open(self.config['data_source'][source]['file'], 'w+', encoding='utf-8') as fd:
response_json = json.dumps(r.text, ensure_ascii=False).encode('utf-8')
fd.write(response_json.decode('utf-8'))
log('JSON from {url} written.'.format(url=self.config['data_source'][source]['url']))
log('JSON file {file} updated.'.format(file=self.config['data_source'][source]['file']))
else:
log('Unable to get JSON: status_code = %d' % r.status_code)
except requests.ConnectionError:
log('Unable to connect to {url}.'.format(url=self.config['data_source']['test_source']['url']))
else:
log('No json_data sources configured: stopping')
self.running = False
def return_all_sports(self):
log('Available sports:')
for i in self.sports:
log(i)
return ','.join(self.sports.keys())
def process_filename(self, name, root):
log('Parsing filename: {name}'.format(name=name))
parsed_name = name.split(self.delimiter, 2)
generated_path = os.path.join(root, parsed_name[0], parsed_name[1])
self.file_dict[name] = os.path.join(root, generated_path)
def return_outcomes_no_names(self, topic_name, bet_types):
outcomes = self.return_outcomes(topic_name, bet_types)
noname_outcomes = outcomes.split('\n')[1]
log('Returning outcomes without names: {noname_outcomes}'.format(noname_outcomes=noname_outcomes))
return noname_outcomes
