def get_public_api_data_daily(environment, sport, date): freeze_list = get_freeze_list(environment) config = get_url_config() public_api_daily_pattern = config[environment]['public-api-base-url'] + config[environment]['public-api-event-list'] public_api_daily_link = public_api_daily_pattern.format(sport=sport, ls_date=date) Log.debug(public_api_daily_link) try: daily = requests.get(public_api_daily_link).json() except Exception: return "Internal Server Error", 500 mapping_template = [] if environment in ["dev", "test"]: mapping_template = Kafka.consume(environment, "export-mapping-template") mapping_template = json.loads(mapping_template[0]) mapping_template = mapping_template["featureProviders"] full_daily = {"Stages": []} for stage in daily["Stages"]: stage["_LC"] = 0 for event in stage["Events"]: if event["Epr"] == 1: stage["_LC"] += 1 if True in (a in freeze_list for a in event["Pids"].values()): event["_FR"] = "1" if environment in ["dev", "test"]: stage["_FP"] = {} stage["_FP"].update(find_priority_rules(mapping_template, get_sport_id_by_name(sport), int(stage["Sid"]))) stage["_FP_unique"] = [] stage["_FP_unique"] = list(set(stage["_FP"].values())) full_daily["Stages"].append(stage) return full_daily
def separate_file(folder, filename): Log.info("Separating file {}".format(filename)) ps_path = 'powershell' separator_path = os.path.join(MODULES_DIR, 'separator.ps1') file = os.path.join(MAIN_REPOSITORY_PATH, folder, filename) command = ps_path + " -Command " + separator_path + " " + file result = subprocess.check_output(command).decode('cp866').rstrip() return result, 200
def remove_old_result_files(): Log.info("Checking RESULTS_DIR for old files") week = 604800 # seconds in 1 week res = list() for file in __listdirattr(RESULTS_DIR): if file['created'] < get_timestamp() - week: Log.info("File {} is older than 1 week. Deleting...".format( file['name'])) delete_file(RESULTS_DIR, file['name']) return res, 200
def connect(dns): Log.info("Connecting to {}".format(dns)) try: ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) ssh.connect(dns, username='******', key_filename='C:\\Users\\IDidyk\\.ssh\\id_rsa.ppk') except TimeoutError as error: Log.error(error) return str(error), 410 else: return ssh
def delete_file(folder, filename): file = os.path.join(MAIN_REPOSITORY_PATH, folder, filename) if os.path.exists(file): os.remove(file) if not os.path.exists(file): Log.info("File {} is deleted".format(filename)) return "File {} removed".format(filename), 200 else: return "Something went wrong", 510 else: return "Not found", 404
def consume(environment, topic): bootstrap_servers = [ "{}-kafka-0.ls.seo:9092".format(environment), "{}-kafka-1.ls.seo:9092".format(environment) ] topic = topic consumer = KafkaConsumer(topic, bootstrap_servers=bootstrap_servers, auto_offset_reset="earliest") for message in consumer: message_str = message.value.decode('utf-8') return message_str, 200 Log.warning("No message gotten from '{}' topic ({})".format( topic, environment)) return "Something went wrong", 500
def move_file(old_folder, new_folder, filename): Log.info("Try to move {} from {} to {}".format(filename, old_folder, new_folder)) old_path = os.path.join(MAIN_REPOSITORY_PATH, old_folder, filename) new_path = os.path.join(MAIN_REPOSITORY_PATH, new_folder, filename) if os.path.exists(old_path): if old_path == new_path: return "File is already in this folder", 400 os.rename(old_path, new_path) if os.path.exists(new_path): return "File moved to /" + new_folder, 200 else: return "Something went wrong", 510 else: return "Not found", 404
def get_freeze_list(environment): if environment in ["dev", "test"]: ids = [] enet_freeze_list_data = {} sr_freeze_list_data = {} freeze_list_data = {} try: enet_freeze_list_data = requests.get("http://" + environment + "-crawler-enetapi-0-lsm.ls-g.net:8070/simulator/listfrozen").json() freeze_list_data += enet_freeze_list_data except Exception: Log.debug("No data about freeze list (ENET)") try: sr_freeze_list_data = requests.get("http://" + environment + "-crawler-sportradar-0-lsm.ls-g.net:8070/simulator/listfrozen").json() freeze_list_data += sr_freeze_list_data except Exception: Log.debug("No data about freeze list (SR)") freeze_list_data = enet_freeze_list_data + sr_freeze_list_data for el in freeze_list_data: id_search = re.search(r"etails\D+(\d+)", el) if id_search: ids.append(id_search.group(1)) return ids else: return []
def collect(data): # Preparation Log.info("Command preparation...") config = get_config() component = config[data['component']] file_name = get_result_file_name(data) ssh = connect(component['dns']) cat_command = get_command(ssh, data, config) # CAT operation Log.info("Command executing...") channel = ssh.get_transport().open_session() channel.exec_command(cat_command) while not channel.exit_status_ready(): time.sleep(1) # Copying from remote to local Log.info("Try to copy remote file...") sftp = ssh.open_sftp() remote_path = "/home/ivan.didyk/{}".format(file_name) local_path = os.path.join(RESULTS_DIR, file_name) sftp.get(remote_path, local_path) # Remove remote file Log.info("Try to remove remote file...") sftp.remove(remote_path) sftp.close() Log.info("SFTP connection closed...") ssh.close() Log.info("SSH connection closed...") return get_result_file_name(data), 200
def get_health_pages(): devtest_url = "http://ls-tools-ls-g.dev-i.net:8091/health-status/" preprod_url = "https://preprod-component-monitoring.livescore.com/health-status/" prod_url = "https://component-monitoring.livescore.com/health-status/" loadtest_url = "http://35.246.114.214:8091/health-status/" loadtest_iron_url = "http://35.246.114.214:8092/health-status/" sr_keys_url = "http://ls-tools-ls-g.dev-i.net:8060/" health_data = [] try: devtest = requests.get(devtest_url).json() for env in devtest: health_data.append({env: devtest[env]}) except Exception as e: Log.warning("Can't get data from {}, {}".format(devtest_url, e)) try: preprod = requests.get(preprod_url).json() for env in preprod: health_data.append({env: preprod[env]}) except Exception as e: Log.warning("Can't get data from {}, {}".format(preprod_url, e)) try: prod = requests.get(prod_url).json() for env in prod: health_data.append({env: prod[env]}) except Exception as e: Log.warning("Can't get data from {}, {}".format(prod_url, e)) try: loadtest = requests.get(loadtest_url).json() for env in loadtest: health_data.append({env: loadtest[env]}) except Exception as e: Log.warning("Can't get data from {}, {}".format(loadtest_url, e)) try: loadtest_iron = { "loadtest-iron": requests.get(loadtest_iron_url).json()['loadtest'] } for env in loadtest_iron: health_data.append({env: loadtest_iron[env]}) except Exception as e: Log.warning("Can't get data from {}, {}".format(loadtest_iron_url, e)) try: tree = html.fromstring(requests.get(sr_keys_url).content) headers = list() lines = list() sr_keys = {"sr-keys": []} for header in tree.xpath("//th/text()"): headers.append(header) for line in tree.xpath("//td/*/text()"): lines.append(line) for i in range(0, len(lines)): if len(sr_keys["sr-keys"]) < i // len(headers) + 1: sr_keys["sr-keys"].append({}) sr_keys["sr-keys"][i // len(headers)].update( {headers[i % len(headers)]: lines[i]}) health_data.append(sr_keys) except Exception as e: Log.warning("Can't get data from {}, {}".format(sr_keys_url, e)) if not os.path.exists(os.path.join(MODULES_DIR, "health_page_cache.txt")): with open(os.path.join(MODULES_DIR, "health_page_cache.txt"), 'w'): pass else: FileManager.write_file(MODULES_DIR, "health_page_cache.txt", json.dumps(health_data)) return health_data, 200