def main():
# basic()
# basic_file()
# best_practice()
from utils import get_module_logger
log = get_module_logger(__name__)
log.info("log module info msg.")
def download_and_process(filename, data_dir):
logger = get_module_logger(__name__)
# need to re-import the logger because of multiprocesing
local_path = download_tfr(filename, data_dir)
process_tfr(local_path, data_dir)
# remove the original tf record to save space
logger.info(f'Deleting {local_path}')
os.remove(local_path)
def main(job_id, params):
from utils import getModel
from utils import getDataset
logger = get_module_logger(__name__)
logger.info('Starting job with id: %d' % job_id)
logger.info('Fetching dataset %s' % params['dataset'])
dataset = getDataset(params['dataset'])()
logger.info('Using model %s' % params['experiment_name'])
simulation = getModel(params['experiment_name'])(**params)
logger.info('Starting training ...')
score = simulation.run(dataset)
return score
#!/usr/bin/python
# -*- coding: utf-8 -*-
import yaml
import re
import time
from sys import exit
from prometheus_client import start_http_server
from prometheus_client.core import GaugeMetricFamily, HistogramMetricFamily, REGISTRY
import utils
from utils import get_module_logger
from consul import Consul
from common import MetricCol, common_metrics_info
logger = get_module_logger(__name__)
class HBaseRegionServerMetricCollector(MetricCol):
def __init__(self, cluster, url):
MetricCol.__init__(self, cluster, url, "hbase", "regionserver")
self._hadoop_regionserver_metrics = {}
for i in range(len(self._file_list)):
self._hadoop_regionserver_metrics.setdefault(
self._file_list[i], {})
def collect(self):
# Request data from ambari Collect Host API
# Request exactly the System level information we need from node
# beans returns a type of 'List'
try:
import utils
from utils import get_driver as driver, remove_ad
log = utils.get_module_logger(__name__)
current_planet = ''
def refresh():
driver().refresh()
def navigate_to_tab(tabName):
current_tab = driver().current_url.split('page=')[1]
if tabName in current_tab:
#Already there
return False
tab = None
menu = driver().find_element_by_id('menuTable')
for element in menu.find_elements_by_tag_name('a'):
try:
if tabName in element.get_attribute('href'):
tab = element
except:
pass
tab.click()
return True
def navigate_to_overview():
return navigate_to_tab('overview')
import os
import sys
import logging
import math
from utils import get_module_logger, ConstraintError
logger = get_module_logger(__name__)
class DerivationRegistry(type):
def __init__(cls, name, bases, cls_dict):
type.__init__(cls, name, bases, cls_dict)
cls._subclasses = set()
for base in bases:
if isinstance(base, DerivationRegistry):
base._subclasses.add(cls)
def getSubclasses(cls):
return list(reduce(
set.union,
(succ.getSubclasses() for succ in cls._subclasses
if isinstance(succ, DerivationRegistry)),
cls._subclasses))
class Simulation(object):
__metaclass__ = DerivationRegistry
