def lookup(self, root): # form a file list to distribute log('Looking for files in {root}...'.format(root=root)) for s in os.listdir(root): if '.' and self.delimiter in s: self.file_dict[s] = '' time.sleep(self.refresh_interval)
def __init__(self, config): self.refresh_interval = config['distribution_service'][ 'refresh_interval'] self.delimiter = config['distribution_service']['delimiter'] self.roots = config['distribution_service']['roots'] self.file_dict = { } # this file list is a dict of a kind 'file name': 'generated destination' log('Distribution service initialized.')
def __init__(self, json_config): self.running = True self.config = json_config self.wait = json_config['wait'] self.xmlrpc_client = xmlrpc.client.ServerProxy( 'http://{host}:{port}/RPC2'.format(host=self.config['server_host'], port=self.config['server_port'])) self.creds = (json_config['olymp_login'], json_config['olymp_password']) log('JSON Updater initialized.')
def __init__(self, config): super(JsonBlazer, self).__init__(config) self.outcome_dict = {} # a dict of a kind 'П1': '2.22' for managing multiple bet types self.json_data = None self.outcomes = {} # outcomes is a list of dicts {'match_name': [{key: value}, {key: value}]} self.sports = {} # sports is a dict of {'football': [{keys: values}, {keys: values}, {keys: values}]} self.json_paths = config['server_data_files'] self.parse_json() log('JSON blazer initialized')
def move_to_new_path(self, path, name, root): # preparing to move original_full_file_path = os.path.join(root, name) new_file_path = os.path.join(root, path) log('Checking dir {dir}...'.format(dir=new_file_path)) os.makedirs(new_file_path, exist_ok=True) # moving here log('Moving {name} to {path}...'.format(name=name, path=path)) shutil.move(original_full_file_path, os.path.join(new_file_path, name))
def return_team_names(self, topic_name, sport_type='Футбол'): # check on empty if len(self.sports) == 0: log('JSON is empty.') return False # determine team names by topic_name for i in self.sports[sport_type]: if i['topic'] == topic_name: return ','.join(i['teams'])
def return_logos(self, sport='Футбол'): # check on empty if len(self.sports) == 0: log('JSON is empty.') return False # call Storage Ravager logo1, logo2 = self.get_team_logos(team1=self.sports['Футбол'][0]['teams'][0], team2=self.sports['Футбол'][0]['teams'][1]) log('Requesting logos by Storage Ravager for: {sport}'.format(sport=sport)) return ','.join([logo1, logo2])
def return_all_sports(self): log('Sport types available:') sport_names = [] for sport in self.sports: sport_names.extend([*sport]) # show found sports log([*sport][0]) # return comma separated string return ','.join(sport_names)
def parse_single_outcome(self, outcome): if outcome: names, values = outcome.split('\n') names_list = names.split(',') values_list = values.split(',') # values and names lists should be same length, else below method wont work if len(names_list) == len(values_list): for name in names_list: self.outcome_dict[name] = values_list[names_list.index(name)] else: log('Length discrepancy: {names} -- {values}'.format(names=names_list, values=values_list)) return False else: log('Unable to parse outcome because its empty.')
def parse_json(self): # to avoid json_data stacking erase existing json_data self.flush() for path in self.json_paths: self.json_data.extend(get_data_file_list(path)) # check if its empty, then parse if len(self.json_data) > 0: for i in self.json_data: if i['sport_type_title'] in self.sports: sub_dict = {'id': i['sport_type_id'], 'topic': i['topic'], 'event_id': str(i['event_id']), 'event_title': i['event_title'], 'teams': (i['team1'], i['team2'],), 'start_date2': i['start_date2'], 'start_date_timestamp': str(i['start_date_timestamp']), 'url': i['url'], 'url_mobile': i['url_mobile'] } self.sports[i['sport_type_title']].append(sub_dict) else: self.sports.update( {i['sport_type_title']: [ {'id': i['sport_type_id'], 'topic': i['topic'], 'event_id': str(i['event_id']), 'event_title': i['event_title'], 'teams': (i['team1'], i['team2'],), 'start_date2': i['start_date2'], 'start_date_timestamp': str(i['start_date_timestamp']), 'url': i['url'], 'url_mobile': i['url_mobile'] } ] } ) sub_dict_outcomes = {i['topic']: i['outcomes']} # if we'll need to find specific outcomes do it here self.outcomes.update(sub_dict_outcomes) return True else: log('JSON file is empty.') return False
def return_team_data(sub_path): # should return available collections content (records) schema, collection = sub_path.split('/', 1) # get schema (database) access db = client[schema] # get a collection (table) access teams = db[collection] # search in collection team_document = teams.find_one({"name_olimp": request.args['team']}) # handle empty result try: return json.dumps(team_document, cls=JSONEncoder) except Exception as e: log('An exception occurred: %s' % e) return str(e)
def run(self): log('Distribution service is running...') while True: for root in self.roots: # update file list self.lookup(root) # generate paths for found files for name in self.file_dict: self.process_filename(name, root) # move files and delete (not copy), log it names_to_delete = [] for name, path in self.file_dict.items(): self.move_to_new_path(path, name, root) names_to_delete.append(name) # deleting processed file from files_dict for name in names_to_delete: log('Removing deleted file name from files dict: %s' % self.file_dict.pop(name, None))
def return_outcomes(self, topic_name, bet_types): # check bet_types if not bet_types: log('No bet types found for {topic}.'.format(topic=topic_name)) return False # ensure that bet_types is actually a several comma-separated types of bets if not ',' in bet_types: if len(bet_types) > 0: bet_types += ',' else: log('Incorrect input: bet_types should be a csv json_data or a single string which is a bet type name.') return False # ensure that outcome_dict is empty if self.outcome_dict: self.outcome_dict = {} # loop via bet types and get all outcomes from each bet type for bet_type in bet_types.split(','): if bet_type: outcome = self.return_outcome(topic_name, bet_type) self.parse_single_outcome(outcome) # parse fresh formed outcomes dict to provide output string of a kind 'П1,П2,Х,П3,П4,Ч;1,2,0,3,4' outcomes = self.prepare_outcomes_string() log('All available bets for {bet_types}: {outcomes}'.format(bet_types=bet_types, outcomes=outcomes)) return outcomes
def get_presenter(self, presenter): # form request url url = 'http://{host}:{port}/api/{database}/Teams'.format(host=self.config['storage_app']['host'], port=self.config['storage_app']['port'], database=self.config['storage_app']['database_name']) payload = {'presenter': presenter} r = requests.get(url, params=payload) if r.status_code != 200: log('Database returned response code %d.' % r.status_code) log('Please check StorageApp log, collection or collection field may not exist.') return r.status_code # parse text response_dict = json.loads(r.text) # get presenter's name presenter_name = response_dict['name'] presenter_position = response_dict['position'] presenter_social = response_dict['vkpage'] # returns string log('Returning presenter info: {name}, position: {position}, social: {social}'.format(name=presenter_name, position=presenter_position, social=presenter_social)) return ','.join([presenter_name, presenter_position, presenter_social])
def get_match_start_date(self, topic_name, sport_type='Футбол'): return_template = 'hh:mm' # check on empty if len(self.outcomes[topic_name]) == 0: log('JSON is empty.') return False timestamp = None # find start_date_timestamp by topic_name for i in self.sports[sport_type]: if i['topic'] == topic_name: timestamp = i['start_date_timestamp'] break # check if timestamp found - return it # if not - return False if timestamp: return parse_13_digits_timestamp(timestamp, return_template) else: log("Unable to find start_date_timestamp: topic mismatch") return False
def return_bet_types(self, topic_name): # check on empty if len(self.outcomes[topic_name]) == 0: log('JSON is empty.') return False # return all bet types for topic_name 'main,nextgoal' try: bet_types = ','.join(self.outcomes[topic_name].keys()) log('Available bet types for {topic}: {bet_types}.'.format(topic=topic_name, bet_types=bet_types)) return bet_types except KeyError: log('Unable to return bet types for %s.' % topic_name) log('No topic %s.' % topic_name) return False
def return_outcome(self, topic_name, bet_type): # return values 'П1,Х,П2;1,2,3' try: bets = {} # fill bets for outcome in self.outcomes[topic_name][bet_type]: bets[outcome['bet_title']] = str(outcome['factor_value']) # now sort it as commented above names = [] values = [] for key, value in bets.items(): names.append(key) values.append(value) bets_in_string = '{bet_titles}\n{values}'.format(bet_titles=','.join(names), values=','.join(values)) log('Available bets for {bet_type}: {bets}'.format(bet_type=bet_type, bets=bets_in_string)) return bets_in_string except KeyError: log('Unable to find outcome for {topic} and {bet_type}.'.format(topic=topic_name, bet_type=bet_type)) return False
def return_all_topics(self, topic='Футбол'): # check on empty if len(self.sports[topic]) == 0: log('JSON is empty.') return False # proceed if not empty topics = [] log('Available events for {topic}:'.format(topic=topic)) for i in self.sports[topic]: topics.append(i['topic']) log(i['topic']) return ','.join(topics)
def get_team_short_name_localized(self, team): # form request url url = 'http://{host}:{port}/api/{database}/Teams'.format(host=self.config['storage_app']['host'], port=self.config['storage_app']['port'], database=self.config['storage_app']['database_name']) payload = {'team': team} r = requests.get(url, params=payload) if r.status_code != 200: log('Database returned response code %d.' % r.status_code) log('Please check StorageApp log, collection or collection field may not exist.') return r.status_code # parse text response_dict = json.loads(r.text) # get logo path name_short_localized = response_dict['name_short_localized'] # returns string log('Returning short name localized: %s' % name_short_localized) return name_short_localized
def return_leagues(self, sport_type): log('Leagues available for %s:' % sport_type) for sport_dict in self.sports: if sport_type in sport_dict.keys(): sport_id = sport_dict[sport_type] break try: league_names = [] for league in self.league: for league_name, league_info in league.items(): if sport_id == league_info[0]: league_names.append(league_name) # return found leagues for l in league_names: log(l) # return comma separated string return ','.join(league_names) except NameError: log('{sport_type} does not exist in current JSON version.'.format(sport_type=sport_type)) return '{sport_type} does not exist in current JSON version.'.format(sport_type=sport_type)
def __init__(self, config): super(StorageRavager, self).__init__() self.config = config log('Storage Ravager initialized.')
@app.route('/') @app.route('/login', methods=['GET', 'POST']) def login(): if request.method == 'POST': # get user/pass from form session['username'] = request.form['username'] session['password'] = request.form['password'] # check user/pass pair # TODO check user via StorageApp # TODO check password via StorageApp for checked user # if check are OK redirect to Main, else abort(403) return redirect(url_for('main')) return render_template('login.html') @app.route('/logout') def logout(): # remove the username from the session if it's there session.pop('username', None) return redirect(url_for('login')) if __name__ == '__main__': # start reaping log('Starting to reap...') app.run(debug=False)
teams = db[collection] # search in collection team_document = teams.find_one({"name_olimp": request.args['team']}) # handle empty result try: return json.dumps(team_document, cls=JSONEncoder) except Exception as e: log('An exception occurred: %s' % e) return str(e) if __name__ == '__main__': # settings # read config json_config = json_reader('config.json') MONGO_HOST = json_config['mongo']['host'] MONGO_PORT = json_config['mongo']['port'] storage_app_host = json_config['storage_app']['host'] storage_app_port = json_config['storage_app']['port'] # init client for 'media' database client = MongoClient(MONGO_HOST, int(MONGO_PORT)) log('Created MongoDB client.') # starting database manager log('Starting database manager...') app.run(debug=False, host=storage_app_host, port=storage_app_port)
try: # prepare server log prepare_logfile() # read config json_config = json_reader('config.json') # Create server host = json_config['server_host'] port = json_config['server_port'] server = SimpleXMLRPCServer((host, int(port))) # Register methods server.register_introspection_functions() server.register_function(get_os, 'get_os') # Register an instance; all the methods of the instance are published as XML-RPC methods server.register_instance(DevourersEater(json_config)) # init json updater and put it in its own thread ju = JsonUpdaterDaemon(json_config) updater_thread = threading.Thread(target=ju.run) updater_thread.start() # Run the server's main loop log('Server is running on {host}:{port}.'.format(host=host, port=port)) server.serve_forever() except KeyboardInterrupt: log('Interrupted from keyboard. Stopped.') sys.exit(0)
def refresh_data(self): self.xmlrpc_client.parse_json() log('In-memory JSON refreshed.')
def update(self): if self.config: for source in self.config['data_source']: log('Requesting json_data source {url}, to be written to "{file}"'.format( url=self.config['data_source'][source]['url'], file=self.config['data_source'][source]['file'])) try: r = requests.get(self.config['data_source'][source]['url'], auth=self.creds) if r.status_code == 200: with open(self.config['data_source'][source]['file'], 'w+', encoding='utf-8') as fd: response_json = json.dumps(r.text, ensure_ascii=False).encode('utf-8') fd.write(response_json.decode('utf-8')) log('JSON from {url} written.'.format(url=self.config['data_source'][source]['url'])) log('JSON file {file} updated.'.format(file=self.config['data_source'][source]['file'])) else: log('Unable to get JSON: status_code = %d' % r.status_code) except requests.ConnectionError: log('Unable to connect to {url}.'.format(url=self.config['data_source']['test_source']['url'])) else: log('No json_data sources configured: stopping') self.running = False
def return_all_sports(self): log('Available sports:') for i in self.sports: log(i) return ','.join(self.sports.keys())
def process_filename(self, name, root): log('Parsing filename: {name}'.format(name=name)) parsed_name = name.split(self.delimiter, 2) generated_path = os.path.join(root, parsed_name[0], parsed_name[1]) self.file_dict[name] = os.path.join(root, generated_path)
def return_outcomes_no_names(self, topic_name, bet_types): outcomes = self.return_outcomes(topic_name, bet_types) noname_outcomes = outcomes.split('\n')[1] log('Returning outcomes without names: {noname_outcomes}'.format(noname_outcomes=noname_outcomes)) return noname_outcomes