def set_config(options, verbosity=0, database=None):
"""Read configuration options from file, merge them with command line
options.
:Parameters:
- 'options': command line options.
- 'verbosity': verbosity level, int.
- 'database': output database name.
:Return:
- 'verbosity': verbosity level, int.
- 'database': output database name, string.
"""
try:
if options.config:
config = ConfigParser.ConfigParser()
config.read(options.config)
verbosity = config.get('task1', 'verbosity')
database = config.get('base', 'database')
if options.verbose:
verbosity = int(options.verbose)
if options.database:
database = options.database
return verbosity, database
except (ConfigParser.Error, Exception) as err:
raise MyError('Configuration error: %s' % err)
def connect_ia(ia_access_key_id=None, ia_secret_access_key=None, is_secure=False, **kwargs):
"""
Connect to the Internet Archive via their S3-like API.
:type ia_access_key_id: string
:param ia_access_key_id: Your IA Access Key ID. This will also look
in your boto config file for an entry in the Credentials
section called "ia_access_key_id"
:type ia_secret_access_key: string
:param ia_secret_access_key: Your IA Secret Access Key. This will also
look in your boto config file for an entry in the Credentials
section called "ia_secret_access_key"
:rtype: :class:`boto.s3.connection.S3Connection`
:return: A connection to the Internet Archive
"""
from boto.s3.connection import S3Connection
from boto.s3.connection import OrdinaryCallingFormat
access_key = config.get("Credentials", "ia_access_key_id", ia_access_key_id)
secret_key = config.get("Credentials", "ia_secret_access_key", ia_secret_access_key)
return S3Connection(
access_key,
secret_key,
host="s3.us.archive.org",
calling_format=OrdinaryCallingFormat(),
is_secure=is_secure,
**kwargs
)
def register_schedule(self):
channel = config.get('irc', 'channel')
for schedule in config.getlist('irc', 'schedules'):
sect = ':'.join(('schedule', schedule))
# do not evaluate isenabled() here.
# if it does, the disabled action will never be scheduled.
if not config.has_section(sect):
logging.error('[schedule] [%s] no such schedule', sect)
continue
if not config.has_option(sect, 'action'):
logging.error('[schedule] [%s] no action specified', sect)
continue
action = ':'.join(('action', config.get(sect, 'action')))
if not config.has_section(action):
logging.error('[schedule] [%s] invalid action specified', sect)
continue
interval = config.getint(sect, 'interval')
if interval < 60:
logging.error('[schedule] [%s] interval too short', sect)
continue
self.reactor.register_schedule(interval, self.do_action,
action, self.connection,
None, {'target': channel}, sect)
logging.info('[schedule] [%s] registered', sect)
def main():
# Set default configuration
port = '8069'
host = 'localhost'
path = expanduser("~") + '/.proxypos/config/proxypos.yaml'
# Read configuration file and init config handler
if os.path.exists(path):
with open(path, 'r') as configfile:
config.import_config(configfile.read())
else:
raise Exception('Please create a default configuration file!')
# Start log file
logging.config.dictConfig(config.get('logs'))
# Interactive mode
logger = logging.getLogger(__name__)
logger.info("ProxyPos server starting up...")
logger.info("Listening on http://%s:%s/" % (config.get('app.host') or host,
config.get('app.port') or port))
run(app,
host=config.get('app.host') or host,
port=config.get('app.port') or port,
quiet=True)
def isenabled(sect, data={}):
if not config.has_section(sect):
return False
if not config.getboolean(sect, 'enable'):
return False
try:
per = config.get(sect, 'percentage') # allow '0'
if per and int(per) < random.randint(1, 100):
return False
time_ = config.get(sect, 'time')
if time_ and not util.time_in(time_):
return False
if 'source' in data:
pattern = config.get(sect, 'source_pattern')
data['source_match'] = re.search(pattern, data['source'])
if not data['source_match']:
return False
if 'message' in data:
pattern = config.get(sect, 'pattern')
data['match'] = re.search(pattern, data['message'])
if not data['match']:
return False
except:
logging.exception('[%s] %s', sect, data)
return False
return True
def configure_logging( config ):
"""
Allow some basic logging configuration to be read from the cherrpy
config.
"""
format = config.get( "log_format", "%(name)s %(levelname)s %(asctime)s %(message)s" )
level = logging._levelNames[ config.get( "log_level", "DEBUG" ) ]
destination = config.get( "log_destination", "stdout" )
log.info( "Logging at '%s' level to '%s'" % ( level, destination ) )
# Get root logger
root = logging.getLogger()
# Set level
root.setLevel( level )
# Remove old handlers
for h in root.handlers[:]:
root.removeHandler(h)
# Create handler
if destination == "stdout":
handler = logging.StreamHandler( sys.stdout )
else:
handler = logging.FileHandler( destination )
# Create formatter
formatter = logging.Formatter( format )
# Hook everything up
handler.setFormatter( formatter )
root.addHandler( handler )
def on_welcome(self, conn, event):
channel = config.get('irc', 'channel')
key = config.get('irc', 'channel_key')
conn.join(channel, key)
logging.info('[welcome] joined <%s>', channel)
self.unregister_schedule()
self.register_schedule()
def load_config(app):
''' Reads the config file and loads configuration properties
into the Flask app.
:param app: The Flask app object.
'''
# Get the path to the application directory,
# that's where the config file resides.
par_dir = os.path.join(__file__, os.pardir)
par_dir_abs_path = os.path.abspath(par_dir)
app_dir = os.path.dirname(par_dir_abs_path)
# Read config file
# FIXME: Use the "common pattern" described in "Configuring from Files":
# http://flask.pocoo.org/docs/config/
config = ConfigParser.RawConfigParser()
config_filepath = app_dir + '/config.cfg'
config.read(config_filepath)
# Set up config properties
app.config['SERVER_PORT'] = config.get('Application', 'SERVER_PORT')
app.config['BASE_PATH'] = config.get('Application', 'BASE_PATH')
app.config['API_ELECTION_RESULTS'] = config.get('Api', 'ELECTION_RESULTS')
# Logging path might be relative or starts from the root.
# If it's relative then be sure to prepend the path with
# the application's root directory path.
log_path = config.get('Logging', 'PATH')
if log_path.startswith('/'):
app.config['LOG_PATH'] = log_path
else:
app.config['LOG_PATH'] = app_dir + '/' + log_path
app.config['LOG_LEVEL'] = config.get('Logging', 'LEVEL').upper()
def __init__(self, **config):
self.config = utils.Config(dict(self.defaults, **config))
logging.config.dictConfig(self.logging_config)
self.log = logging.getLogger('irc3.' + self.nick)
self.original_nick = self.nick
if config.get('verbose'):
logging.getLogger('irc3').setLevel(logging.DEBUG)
else:
level = config.get('level')
if level is not None:
level = getattr(logging, str(level), level)
self.log.setLevel(level)
self.encoding = self.config['encoding']
self.loop = self.config.loop
if self.loop is None:
self.loop = asyncio.get_event_loop()
self.events_re = []
self.events = defaultdict(list)
self.plugins = {}
self.includes = set()
self.include(*self.config.get('includes', []))
self.recompile()
def __init__(self, *ini, **config):
config['version'] = version
self.config = utils.Config(dict(self.defaults, *ini, **config))
logging.config.dictConfig(self.logging_config)
if self.server:
self.log = logging.getLogger('irc3d')
else:
self.log = logging.getLogger('irc3.' + (self.nick or 'd'))
self.original_nick = self.nick
if config.get('verbose') or config.get('debug'):
logging.getLogger('irc3').setLevel(logging.DEBUG)
logging.getLogger('irc3d').setLevel(logging.DEBUG)
else:
level = config.get('level')
if level is not None:
level = getattr(logging, str(level), level)
self.log.setLevel(level)
self.encoding = self.config['encoding']
self.loop = self.config.loop
if self.loop is None:
try:
self.loop = asyncio.get_event_loop()
except RuntimeError:
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(self.loop)
# python 3.4.1 do not have a create_task method. check for it
self.create_task = getattr(self.loop, 'create_task', self.create_task)
self.registry = Registry()
self.include(*self.config.get('includes', []))
def main(options, config):
monitored_pages = list()
page_monitors = list()
if options.web_server:
launch_web_server(monitored_pages)
for section in config.sections():
page = MonitoredPage(section,
config.get(section, 'url'),
float(config.get(section, 'maximum_response_time')),
config.get(section, 'should_contain'))
monitored_pages.append(page)
page_monitors.append(monitor_page(page, float(options.frequency)))
try:
while True:
for page in monitored_pages:
try:
printable_status = str(page)
if page.get_status():
logger.info(printable_status)
else:
logger.warning(printable_status)
except NoResponse:
logger.error(printable_status)
time.sleep(options.frequency)
except KeyboardInterrupt:
logger.info('Ctrl-C caught, exiting.')
map(lambda x: x.cancel(), page_monitors)
map(lambda x: x.join(), page_monitors)
return 0
def read_sequence_configs(config):
"""[EXTERNAL_DIRS]."""
CONFIGS['sequence_dir'] = config.get(
'sequence_dir',
fallback=op.join(CONFIGS['unique_temp_dir'], 'sequence')
)
CONFIGS['provean_temp_dir'] = op.join(CONFIGS['sequence_dir'], 'provean_temp')
_validate_provean_temp_dir(config, CONFIGS)
CONFIGS['pdb_dir'] = config.get('pdb_dir')
CONFIGS['blast_db_dir'] = config.get('blast_db_dir')
CONFIGS['blast_db_dir_fallback'] = (
config.get('blast_db_dir_fallback', fallback=''))
_validate_blast_db_dir(CONFIGS)
CONFIGS['archive_dir'] = config.get('archive_dir')
# Supported archive types are 'directory' and '7zip'
if CONFIGS['archive_dir'] is None:
CONFIGS['archive_type'] = None
elif op.splitext(CONFIGS['archive_dir'])[-1] in ['.7z', '.7zip']:
assert op.isfile(CONFIGS['archive_dir'])
CONFIGS['archive_type'] = '7zip'
else:
assert op.isdir(CONFIGS['archive_dir'])
CONFIGS['archive_type'] = 'directory'
CONFIGS['archive_temp_dir'] = op.join(CONFIGS['temp_dir'], 'archive')
def configure_logging( config ):
"""
Allow some basic logging configuration to be read from the cherrpy
config.
"""
# PasteScript will have already configured the logger if the appropriate
# sections were found in the config file, so we do nothing if the
# config has a loggers section, otherwise we do some simple setup
# using the 'log_*' values from the config.
if config.global_conf_parser.has_section( "loggers" ):
return
format = config.get( "log_format", "%(name)s %(levelname)s %(asctime)s %(message)s" )
level = logging._levelNames[ config.get( "log_level", "DEBUG" ) ]
destination = config.get( "log_destination", "stdout" )
log.info( "Logging at '%s' level to '%s'" % ( level, destination ) )
# Get root logger
root = logging.getLogger()
# Set level
root.setLevel( level )
# Turn down paste httpserver logging
if level <= logging.DEBUG:
logging.getLogger( "paste.httpserver.ThreadPool" ).setLevel( logging.WARN )
# Remove old handlers
for h in root.handlers[:]:
root.removeHandler(h)
# Create handler
if destination == "stdout":
handler = logging.StreamHandler( sys.stdout )
else:
handler = logging.FileHandler( destination )
# Create formatter
formatter = logging.Formatter( format )
# Hook everything up
handler.setFormatter( formatter )
root.addHandler( handler )
def main():
args = make_args()
config = configparser.ConfigParser()
utils.load_config(config, args.config)
for cmd in args.modify:
utils.modify_config(config, cmd)
with open(os.path.expanduser(os.path.expandvars(args.logging)), 'r') as f:
logging.config.dictConfig(yaml.load(f))
model_dir = utils.get_model_dir(config)
height, width = tuple(map(int, config.get('image', 'size').split()))
resize = transform.parse_transform(config, config.get('transform', 'resize_test'))
transform_image = transform.get_transform(config, config.get('transform', 'image_test').split())
transform_tensor = transform.get_transform(config, config.get('transform', 'tensor').split())
# load image
image_bgr = cv2.imread('image.jpg')
image_resized = resize(image_bgr, height, width)
image = transform_image(image_resized)
tensor = transform_tensor(image).unsqueeze(0)
# Caffe2
init_net = caffe2_pb2.NetDef()
with open(os.path.join(model_dir, 'init_net.pb'), 'rb') as f:
init_net.ParseFromString(f.read())
predict_net = caffe2_pb2.NetDef()
with open(os.path.join(model_dir, 'predict_net.pb'), 'rb') as f:
predict_net.ParseFromString(f.read())
p = workspace.Predictor(init_net, predict_net)
results = p.run([tensor.numpy()])
logging.info(utils.abs_mean(results[0]))
logging.info(hashlib.md5(results[0].tostring()).hexdigest())
def __init__(self, config):
self.command = ''
self.jd_workflow = None
self.param = []
self.log_file = config.get("scheduler", "log_file")
self.log_level = config.get("scheduler", "log_level")
self.workflow_not_deployed = config.get("scheduler", "workflow_not_deployed")
def setup_logging(default_level=logging.WARNING):
"""Setup logging configuration
"""
log = logging.getLogger(__name__)
config = ConfigParser.RawConfigParser()
fullPath = LiotaConfigPath().get_liota_fullpath()
if fullPath != '':
try:
if config.read(fullPath) != []:
# now use json file for logging settings
try:
log_path = config.get('LOG_PATH', 'log_path')
log_cfg = config.get('LOG_CFG', 'json_path')
except ConfigParser.ParsingError as err:
log.error('Could not parse log config file')
else:
raise IOError('Cannot open configuration file ' + fullPath)
except IOError as err:
log.error('Could not open log config file')
mkdir_log(log_path)
if os.path.exists(log_cfg):
with open(log_cfg, 'rt') as f:
config = json.load(f)
logging.config.dictConfig(config)
log.info('created logger with ' + log_cfg)
else:
# missing logging.json file
logging.basicConfig(level=default_level)
log.warn(
'logging.json file missing,created default logger with level = ' +
str(default_level))
else:
# missing config file
log.warn('liota.conf file missing')
def setup_console_logging(config, verbosity_level):
if verbosity_level < min(LOG_LEVELS.keys()):
verbosity_level = min(LOG_LEVELS.keys())
if verbosity_level > max(LOG_LEVELS.keys()):
verbosity_level = max(LOG_LEVELS.keys())
loglevels = config.get('loglevels', {})
has_debug_loglevels = any([
level < logging.INFO for level in loglevels.values()])
verbosity_filter = VerbosityFilter(verbosity_level, loglevels)
if verbosity_level < 1 and not has_debug_loglevels:
log_format = config['logging']['console_format']
else:
log_format = config['logging']['debug_format']
formatter = logging.Formatter(log_format)
if config['logging']['color']:
handler = ColorizingStreamHandler(config.get('logcolors', {}))
else:
handler = logging.StreamHandler()
handler.addFilter(verbosity_filter)
handler.setFormatter(formatter)
logging.getLogger('').addHandler(handler)
def check_config():
"""
Check the config file and print results
"""
result = {'missing': [], 'changed': []}
for option in DEFAULT_CONFIG['Poezio']:
value = config.get(option)
if value != DEFAULT_CONFIG['Poezio'][option]:
result['changed'].append((option, value, DEFAULT_CONFIG['Poezio'][option]))
else:
value = config.get(option, default='')
upper = value.upper()
default = str(DEFAULT_CONFIG['Poezio'][option]).upper()
if upper != default:
result['missing'].append(option)
result['changed'].sort(key=lambda x: x[0])
result['missing'].sort()
if result['changed']:
print('\033[1mOptions changed from the default configuration:\033[0m\n')
for option, new_value, default in result['changed']:
print(' \033[1m%s\033[0m = \033[33m%s\033[0m (default: \033[32m%s\033[0m)' % (option, new_value, default))
if result['missing']:
print('\n\033[1mMissing options:\033[0m (the defaults are used)\n')
for option in result['missing']:
print(' \033[31m%s\033[0m' % option)
def _makeConfig(self):
import datetime
config = deployUtil.vdsmImport("config").config
if not os.path.exists(VDSM_CONF):
logging.debug("makeConfig: generating conf.")
lines = []
lines.append("# Auto-generated by vds_bootstrap at:" + str(datetime.datetime.now()) + "\n")
lines.append("\n")
lines.append("[vars]\n") # Adding ts for the coming scripts.
lines.append("trust_store_path = " + config.get("vars", "trust_store_path") + "\n")
lines.append("ssl = " + config.get("vars", "ssl") + "\n")
if config.getboolean("vars", "fake_kvm_support"):
lines.append("fake_kvm_support = true\n")
lines.append("\n")
# Adding mgt port for the coming scripts.
lines.append("[addresses]\n")
lines.append("management_port = " + config.get("addresses", "management_port") + "\n")
logging.debug("makeConfig: writing the following to " + VDSM_CONF)
logging.debug(lines)
fd, tmpName = tempfile.mkstemp()
f = os.fdopen(fd, "w")
f.writelines(lines)
f.close()
os.chmod(tmpName, 0o644)
shutil.move(tmpName, VDSM_CONF)
else:
self.message = "Basic configuration found, skipping this step"
logging.debug(self.message)
def get_window_size(self, tenantId):
"""
This method is in charge of retrieve the window size of a tenantId from cloto database.
:param tenantId: the id of the tenant to request the windowsize
:return: the window size
"""
try:
if self.conn == None:
self.conn = mysql.connect(
charset=config.get("mysql", "charset"),
use_unicode=True,
host=config.get("mysql", "host"),
user=config.get("mysql", "user"),
passwd=config.get("mysql", "password"),
db=config.get("mysql", "db"),
)
cursor = self.conn.cursor()
cursor.execute('SELECT * FROM cloto.cloto_tenantinfo WHERE tenantId="%s"' % tenantId)
data = cursor.fetchall()
if len(data) == 0:
raise NotFound('{"error": "TenantID %s not found in cloto database"}' % tenantId)
else:
tenant_information = data[0]
window_size = tenant_information[1]
except Exception, e:
logging.error("Error %s" % e.message)
raise e
def main():
home = os.path.dirname(os.path.abspath(__file__))
os.chdir(home)
config = ConfigParser.RawConfigParser()
config.read('application.conf')
out_dir = config.get('application', 'out_dir')
timeout = float(config.get('application', 'timeout'))
boxes = json.loads(config.get('application', 'boxes_json'))
logging.config.fileConfig('logging.conf')
setup_encoding()
for box in boxes:
host = box['host']
username = box['username']
password = box['password']
try:
logging.info('Processing %s', host)
api = connect(host=host, username=username, password=password, timeout=timeout)
entries = api(cmd='/ip/arp/print')
api.close()
data = set([(entry['mac-address'], entry['interface']) for entry in entries if 'mac-address' in entry])
file_name = datetime.now().strftime('%Y%m%d%H%M%S') + '-' + host + '.gz'
file_path = os.path.join(out_dir, file_name)
with gzip.open(file_path, 'wb') as file:
for obj in data:
file.write(' '.join(obj) + '\n')
logging.info('%s done', host)
except:
_, exc_value, _ = sys.exc_info()
logging.error(exc_value)
def transcode_remote():
setup_logging()
config = get_config()
args = sys.argv[1:]
# Check to see if we need to call a user-script to replace/modify the file path
if config.get("path_script", None):
idx = 0
# The file path comes after the "-i" command line argument
for i, v in enumerate(args):
if v == "-i":
idx = i+1
break
# Found the requested video path
path = args[idx]
try:
proc = subprocess.Popen([config.get("path_script"), path], stdout=subprocess.PIPE)
proc.wait()
new_path = proc.stdout.readline().strip()
if new_path:
log.debug("Replacing path with: %s" % new_path)
args[idx] = new_path
except Exception, e:
log.error("Error calling path_script: %s" % str(e))
def enabled(self, sect):
if not config.has_section(sect):
return False
if not config.getboolean(sect, 'enable'):
return False
try:
per = config.get(sect, 'percentage') # allow '0'
if per and int(per) < random.randint(1, 100):
return False
time_ = config.get(sect, 'time')
if time_ and not util.time_in(time_):
return False
if 'user_name' in self.data:
pattern = config.get(sect, 'user_pattern')
self.data['user_match'] = re.search(pattern,
self.data['user_name'])
if not self.data['user_match']:
return False
if 'text' in self.data:
pattern = config.get(sect, 'pattern')
self.data['match'] = re.search(pattern, self.data['text'])
if not self.data['match']:
return False
except:
logging.exception('[%s] %s', sect, self.data)
return False
return True
def load_config():
"""
Loads the config files merging the defaults
with the file defined in environ.LINTREVIEW_SETTINGS if it exists.
"""
config = Config(os.getcwd())
if 'LINTREVIEW_SETTINGS' in os.environ:
config.from_envvar('LINTREVIEW_SETTINGS')
elif os.path.exists(os.path.join(os.getcwd(), 'settings.py')):
config.from_pyfile('settings.py')
else:
msg = ("Unable to load configuration file. Please "
"either create ./settings.py or set LINTREVIEW_SETTINGS "
"in your environment before running.")
raise ImportError(msg)
if config.get('LOGGING_CONFIG'):
logging.config.fileConfig(
config.get('LOGGING_CONFIG'),
disable_existing_loggers=False)
if config.get('SSL_CA_BUNDLE'):
os.environ['REQUESTS_CA_BUNDLE'] = config.get('SSL_CA_BUNDLE')
return config
def phase_selfcal(field='',vis='',refant='',config=None):
"""
Self-calibrate phases
Inputs:
field = field to be calibrated
vis = measurement set containing field
config = ConfigParser object for this project
Returns:
None
"""
#
# start logger
#
logger = logging.getLogger("main")
#
# check config
#
if config is None:
logger.critical("Error: Need to supply a config")
raise ValueError("Config is None")
#
# Calculate phase calibration table
#
solint = config.get("Self Calibration","solint")
combine = config.get("Self Calibration","combine")
logger.info("Calculate phase calibration table with solint={0} and combine={1}".format(solint,combine))
casa.gaincal(vis=vis,field=field,caltable='{0}_phase_selfcal.cal'.format(field),
calmode='p',solint=solint,combine=combine,
refant=refant,minsnr=3.0,minblperant=1)
logger.info("Done.")
def load_modules():
analysis_modules = []
for section in config:
if "analysis_module_" in section:
if not config.getboolean(section, "enabled"):
continue
module_name = config.get(section, "module")
try:
_module = importlib.import_module(module_name)
except Exception as e:
log.error("Unable to import module {0}: {1}".format(module_name, str(e)))
continue
class_name = config.get(section, "class")
try:
module_class = getattr(_module, class_name)
except Exception as e:
log.error("Unable to load module class {0}: {1}".format(module_class, str(e)))
continue
try:
analysis_module = module_class(str(section))
except Exception as e:
log.error("Unable to load analysis module {0}: {1}".format(section, str(e)))
continue
analysis_modules.append(analysis_module)
return analysis_modules
def udl_trigger(config, loop_once=False):
"""Runs the watcher script on the udl arrivals zone to schedule pipeline
when a file is ready
:param config: Entire udl2_conf as flat dictionary
:pram loop_once: Runs the loop only once if set to True (Needed for testing)
"""
# get the settings needed for the udl trigger alone
config = get_config_from_ini(config=config, config_prefix='udl2_trigger', delete_prefix=True)
file_watcher = FileWatcher(config, append_logs_to='edudl2')
logger.info('Starting UDL2 trigger loop. Looking at directory => {source_dir}'.format(
source_dir=config.get(Const.SOURCE_DIR)))
while True:
try:
logger.debug('Searching for new files in {source_dir}'.format(source_dir=config.get(Const.SOURCE_DIR)))
udl_ready_files = _find_udl_ready_files(file_watcher)
logger.debug('Found {count} files ready to process'.format(count=str(len(udl_ready_files))))
for file in udl_ready_files:
logger.debug('Scheduling pipeline for file - {file}'.format(file=file))
schedule_pipeline.delay(file)
if loop_once:
break
except KeyboardInterrupt:
logger.warn('UDL2 trigger process terminated by a user')
os._exit(0)
except Exception as e:
logger.error(e)
finally:
time.sleep(float(file_watcher.conf.get(Const.FILE_SYSTEM_SCAN_DELAY)))
logger.warn('Exiting udl trigger process')
def reset():
global bin_paths
global config
global configs_found
bin_paths = {}
config = ConfigParser.ConfigParser()
configs_found = config.read(config_files)
if not configs_found:
print ("WARNING: pyTivo.conf does not exist.\n" + "Assuming default values.")
configs_found = config_files[-1:]
for section in config.sections():
if section.startswith("_tivo_"):
tsn = section[6:]
if tsn.upper() not in ["SD", "HD"]:
if config.has_option(section, "name"):
tivo_names[tsn] = config.get(section, "name")
else:
tivo_names[tsn] = tsn
if config.has_option(section, "address"):
tivos[tsn] = config.get(section, "address")
for section in ["Server", "_tivo_SD", "_tivo_HD"]:
if not config.has_section(section):
config.add_section(section)
def client_from_config(self, queue_name, config):
"""
Return an XQueueClient from the configuration object.
"""
from . import client
klass = getattr(client, config.get('CLASS', 'XQueueClientThread'))
watcher = klass(queue_name,
xqueue_server=config.get('SERVER', 'http://localhost:18040'),
xqueue_auth=config.get('AUTH', (None, None)),
http_basic_auth=self.http_basic_auth)
for handler_config in config.get('HANDLERS', []):
handler_name = handler_config['HANDLER']
mod_name, classname = handler_name.rsplit('.', 1)
module = importlib.import_module(mod_name)
kw = handler_config.get('KWARGS', {})
# codejail configuration per handler
codejail_config = handler_config.get("CODEJAIL", None)
if codejail_config:
kw['codejail_python'] = self.enable_codejail(codejail_config)
handler = getattr(module, classname)
if kw or inspect.isclass(handler):
# handler could be a function or a class
handler = handler(**kw)
watcher.add_handler(handler)
return watcher
def __init__(self, args, config):
self.args = args
self.config = config
self.cache_dir = utils.get_cache_dir(config)
self.model_dir = utils.get_model_dir(config)
self.category = utils.get_category(config, self.cache_dir if os.path.exists(self.cache_dir) else None)
self.draw_bbox = utils.visualize.DrawBBox(config, self.category)
self.anchors = torch.from_numpy(utils.get_anchors(config)).contiguous()
self.height, self.width = tuple(map(int, config.get('image', 'size').split()))
self.path, self.step, self.epoch = utils.train.load_model(self.model_dir)
state_dict = torch.load(self.path, map_location=lambda storage, loc: storage)
self.dnn = utils.parse_attr(config.get('model', 'dnn'))(model.ConfigChannels(config, state_dict), self.anchors, len(self.category))
self.dnn.load_state_dict(state_dict)
self.inference = model.Inference(config, self.dnn, self.anchors)
self.inference.eval()
if torch.cuda.is_available():
self.inference.cuda()
logging.info(humanize.naturalsize(sum(var.cpu().numpy().nbytes for var in self.inference.state_dict().values())))
self.create_cap()
self.create_cap_size()
self.create_writer()
self.keys = set(args.keys)
self.resize = transform.parse_transform(config, config.get('transform', 'resize_test'))
self.transform_image = transform.get_transform(config, config.get('transform', 'image_test').split())
self.transform_tensor = transform.get_transform(config, config.get('transform', 'tensor').split())
def main():
if len(sys.argv) < 2 or '-h' in sys.argv:
print("Usage: update_orgs etc/search.ini [custom_index_names]")
sys.exit(1)
parser = ConfigParser()
parser.read(sys.argv[1])
config = dict(parser.items('search_engine'))
config = decode_bool_values(config)
uo_config = dict(parser.items('update_orgs'))
if len(sys.argv) > 2:
config['index_names'] = sys.argv[2]
logging.config.fileConfig(sys.argv[1])
logger.info("Starting openprocurement.search.update_orgs v%s", __version__)
logger.info("Copyright (c) 2015-2018 Volodymyr Flonts <*****@*****.**>")
# try get exclusive lock to prevent second start
lock_filename = uo_config.get('pidfile') or 'update_orgs.pid'
lock_file = open(lock_filename, "w")
fcntl.lockf(lock_file, fcntl.LOCK_EX + fcntl.LOCK_NB)
lock_file.write(str(os.getpid()) + "\n")
lock_file.flush()
signal.signal(signal.SIGTERM, sigterm_handler)
# signal.signal(signal.SIGINT, sigterm_handler)
try:
chage_process_user_group(config)
except Exception as e:
logger.error("Can't change process user: %s", str(e))
try:
global engine
engine = IndexOrgsEngine(config, uo_config)
source = OrgsSource(config)
index = OrgsIndex(engine, source, config)
# manualy reset and prevent second reset on first process_source
source.reset()
index.last_current_index = index.current_index
if config.get('tender_api_url', None):
source = TenderSource(config)
engine.process_source(source)
if config.get('ocds_dir', None):
source = OcdsSource(config)
engine.process_source(source)
if config.get('plan_api_url', None):
source = PlanSource(config)
engine.process_source(source)
if config.get('auction_api_url', None):
source = AuctionSource(config)
engine.process_source(source)
engine.flush_orgs_map()
except Exception as e:
logger.exception("Exception: %s", str(e))
finally:
lock_file.close()
os.remove(lock_filename)
logger.info("Shutdown")
return 0
def read_config(self, config):
self.verbosity = config.get("verbose", 0)
self.log_config = self.abspath(config.get("log_config"))
self.log_file = self.abspath(config.get("log_file"))
if config.get("full_twisted_stacktraces"):
debug_deferreds()
def generate_files(self, config):
log_config = config.get("log_config")
if log_config and not os.path.exists(log_config):
with open(log_config, "wb") as log_config_file:
log_config_file.write(
DEFAULT_LOG_CONFIG.substitute(log_file=config["log_file"]))
'formatters': {
'simple': {
'format': '[%(levelname)s/%(name)s] %(message)s',
},
},
'handlers': {
'console': {
'level': 'DEBUG',
'class': 'logging.StreamHandler',
'formatter': 'simple',
},
},
'loggers': {
'tunneldigger': {
'handlers': ['console'],
'level': config.get('log', 'verbosity'),
}
}
}
logging.config.dictConfig(LOGGING_CONFIGURATION)
# Logger.
logger = logging.getLogger("tunneldigger.broker")
logger.info("Initializing the tunneldigger broker.")
# Initialize the event loop.
event_loop = eventloop.EventLoop()
# Initialize the hook manager.
hook_manager = hooks.HookManager(
event_loop=event_loop,
host_name = str(sys.argv[1])
host_invMode = str(sys.argv[2])
host_id = 11008
#host_id = str(host_id)
print "\n---------------------------"
print "host_name: %s" % host_name
print "host_invMode: %s" % host_invMode
print "---------------------------\n"
# GET LOCAL CONFIGURATION
config = get_config("zdh.conf")
server = {}
server['url'] = config.get('server', 'url')
server['user'] = config.get('server', 'user')
server['password'] = config.get('server', 'password')
logger.debug("server = '%s'" % server)
# LOGIN TO ZABBIX
zapi = ZabbixAPI(server['url'])
zapi.login(server['user'], server['password'])
logger.debug("connected to Zabbix API Version %s" % zapi.api_version())
#zapi.host.massupdate(hosts=hlookup,inventory_mode=invm)
#11008
if host_invMode == "auto": host_invMode = int(1)
elif host_invMode == "disabled": host_invMode = int(-1)
def __init__(self, zookeeper, **kwargs):
"""
:type zookeeper: kazoo.client.KazooClient
"""
self._zookeeper = zookeeper
self._settings = kwargs
try:
data, stat = self._zookeeper.get(ZOOM_CONFIG)
config = json.loads(data)
# create 'logs' directory if it does not exist
if not os.path.exists("logs"):
os.makedirs("logs")
# initialize logging
logging_config = config.get('logging')
logging.config.dictConfig(logging_config)
# get system type
running_os = self._get_system()
self._host = socket.gethostname()
# web_server
web_server_settings = config.get('web_server')
self._port = self._get_setting('port',
web_server_settings.get('port'))
self._is_debug = web_server_settings.get('debug')
self._application_path = os.getcwd()
self._client_path = zk_path_join(
(os.path.normpath(os.getcwd() + os.sep + os.pardir)), 'client')
self._doc_path = zk_path_join(
(os.path.normpath(os.getcwd() + os.sep + os.pardir)), "doc")
self._html_path = zk_path_join(self._client_path, "views")
self._images_path = zk_path_join(self._client_path, "images")
self._pid = os.getpid()
self._environment = self._get_setting(
'environment', os.environ.get('EnvironmentToUse', 'Staging'))
# zookeeper
zookeeper_settings = config.get('zookeeper')
self._zookeeper_paths = zookeeper_settings
self._agent_configuration_path = zookeeper_settings.get(
'agent_configuration_path')
self._agent_state_path = zookeeper_settings.get('agent_state_path')
self._task_path = zookeeper_settings.get('task_path')
self._application_state_path = zookeeper_settings.get(
'application_state_path')
self._global_mode_path = zookeeper_settings.get('global_mode_path')
self._pillar_path = zookeeper_settings.get('pillar_path')
self._alert_path = zookeeper_settings.get('alert_path')
self._override_node = zookeeper_settings.get(
'override_node', '/spot/software/config/override')
self._temp_directory = zookeeper_settings.get('temp_directory')
self._zookeeper_host = get_zk_conn_string(self._environment)
# pagerduty
pagerduty_settings = config.get('pagerduty')
self._pagerduty_default_svc_key = pagerduty_settings.get(
'pagerduty_default_svc_key')
self._pagerduty_api_token = pagerduty_settings.get(
'pagerduty_api_token')
self._pagerduty_subdomain = pagerduty_settings.get(
'pagerduty_subdomain')
self._pagerduty_enabled_environments = pagerduty_settings.get(
'pagerduty_enabled_environments')
self._pagerduty_alert_footer = pagerduty_settings.get(
'pagerduty_footer', '')
# database
db_settings = config.get('database')
self._db_type = db_settings.get('db_type')
if running_os == PlatformType.WINDOWS:
self._sql_connection = db_settings.get(
'sql_connection_windows')
else:
self._sql_connection = db_settings.get('sql_connection')
# authentication
ad_settings = config.get('active_directory')
self._ldap_server = ad_settings.get('host')
self._ldap_port = ad_settings.get('port')
# environment specific
env_settings = config.get(self._environment.lower())
self._read_write_groups = env_settings.get('read_write_groups')
self._graphite_host = env_settings.get('graphite_host')
self._graphite_recheck = env_settings.get('graphite_recheck', '5m')
# chatops
chatops_settings = env_settings.get('chatops', {})
self._chatops_url = chatops_settings.get('url')
self._chatops_group = chatops_settings.get('group')
self._chatops_commands_to_chat = chatops_settings.get(
'commands_to_chat')
# message throttling
throttle_settings = config.get('message_throttle')
self._throttle_interval = throttle_settings.get('interval')
# salt
self._salt_settings = env_settings.get('saltREST')
except ValueError as e:
logging.error('Data at {0} is not valid JSON.'.format(ZOOM_CONFIG))
raise e
except NoNodeError as e:
logging.error('Config node missing: {}'.format(ZOOM_CONFIG))
raise e
except Exception as e:
logging.exception('An unhandled exception occurred.')
raise e
keyVals = options.substitutions.split(",")
for keyVal in keyVals:
parts = keyVal.split(":", 1)
subsDict[parts[0]] = parts[1]
if (logger != None):
logger.debug("Substitution Key: %s Value: %s" %
(parts[0], parts[1]))
emailServerDict = {}
config = ConfigParser.RawConfigParser()
config.read(options.templateFile)
emailServer = config.items("EmailServer")
for entry in emailServer:
key = entry[0]
emailServerDict[key] = entry[1]
recipients = config.get("To", "Recipients")
recipients = recipients.split(",")
messageNfoDict = {}
messageNfo = config.items("Email")
for entry in messageNfo:
key = entry[0]
messageNfoDict[key] = entry[1]
smtp = smtpClass(emailServerDict['server'], emailServerDict['from'],
emailServerDict['pwd'])
messageNfoDict['attachments'] = messageNfoDict['attachments'] % (
subsDict)
fileAttachemnts = messageNfoDict['attachments'].split(",")
for filePath in fileAttachemnts:
filePath = filePath.replace('"', '')
import logging.config
import os
from bamt.config import config
import warnings
log_file_path = config.get('LOG',
'log_conf_loc',
fallback='log_conf_path is not defined')
if not os.path.isdir(os.path.join(os.path.expanduser("~"), 'BAMT')):
os.mkdir(os.path.join(os.path.expanduser("~"), 'BAMT'))
try:
logging.config.fileConfig(log_file_path)
except:
log_file_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'logging.conf')
logging.config.fileConfig(log_file_path)
warnings.warn(
"Reading log path location from config file failed. Default location will be used instead."
)
logger_builder = logging.getLogger('builder')
logger_network = logging.getLogger('network')
logger_preprocessor = logging.getLogger('preprocessor')
logger_nodes = logging.getLogger('nodes')
logging.captureWarnings(True)
logger_warnings = logging.getLogger('py.warnings')
def _ws_single(request, context):
"""
Single Row Processing for Websockets
:param request: iterable sequence of bundled rows
:return: the same iterable sequence as received
"""
logging.info('Entering {} TimeStamp: {}'.format(
function_name,
datetime.now().strftime("%H:%M:%S.%f")))
# Start by Gathering Environmental Varaiable
host = socket.gethostname()
ip_addr = socket.gethostbyname(host)
ws_url = config.get(q_function_name, 'ws_url')
token = config.get(q_function_name, 'token')
user_name = config.get(q_function_name, 'username')
ws_route = config.get(q_function_name, 'ws_route')
bCache = config.get(q_function_name, 'cache')
logging.debug('Pringint Route for WS {}'.format(ws_route))
logging.debug("Caching is set to {}".format(bCache))
if (bCache.lower() == "true"):
logging.info(
"Caching ****Enabled*** for {}".format(q_function_name))
else:
logging.info(
"Caching ****Disabled**** for {}".format(q_function_name))
md = (('qlik-cache', 'no-store'), )
context.send_initial_metadata(md)
# In Future we will use the Token for Liencensing and Throttling
# Currently we are using Comblination of host+ipaddr+username for Client Identification
ws_url = ws_url + host + '_' + ip_addr + '_' + user_name + '_'
logging.debug('Websocket URL : {}'.format(ws_url))
ws = create_connection(ws_url)
response_rows = []
for request_rows in request:
# Iterate over rows
# Default code
for row in request_rows.rows:
# Retrieve string value of parameter and append to the params variable
# Length of param is 1 since one column is received, the [0] collects the first value in the list
param = [d.strData for d in row.duals][0]
result = ''
if (len(param) == 0):
logging.debug('Parameters are Empty')
result = 'Error'
else:
payload = '{"action":"' + ws_route + \
'","data":"' + param + '"}'
logging.debug('Showing Payload: {}'.format(payload))
ws.send(payload)
#logging.info('Show Payload Response: {}'.format(resp.text))
resp = json.loads(ws.recv())
logging.debug(resp)
result = resp['result']
logging.debug('Show Result: {}'.format(result))
# Create an iterable of dual with the result
duals = iter([SSE.Dual(strData=result)])
response_rows.append(SSE.Row(duals=duals))
# Yield the row data as bundled rows
yield SSE.BundledRows(rows=response_rows)
ws.close()
logging.info('Exiting {} TimeStamp: {}'.format(
function_name,
datetime.now().strftime("%H:%M:%S.%f")))
def main():
'''Main(). Commandline parsing and stalker startup.'''
parser = argparse.ArgumentParser()
parser.add_argument("-p",
"--posttroll_port",
dest="posttroll_port",
help="Local port where messages are published")
parser.add_argument("-t",
"--topic",
dest="topic",
help="Topic of the sent messages")
parser.add_argument("-c",
"--configuration_file",
help="Name of the config.ini configuration file")
parser.add_argument("-C",
"--config_item",
help="Name of the configuration item to use")
parser.add_argument("-e",
"--event_names",
help="Name of the events to monitor")
parser.add_argument("-f",
"--filepattern",
help="Filepath pattern used to parse "
"satellite/orbit/date/etc information")
parser.add_argument("-i",
"--instrument",
help="Instrument name in the satellite")
if len(sys.argv) <= 1:
parser.print_help()
sys.exit()
else:
args = parser.parse_args()
# Parse commandline arguments. If args are given, they override
# the configuration file.
args_dict = vars(args)
args_dict = {k: args_dict[k] for k in args_dict if args_dict[k] != None}
config = {}
if args.configuration_file is not None:
config_fname = args.configuration_file
if "template" in config_fname:
print("Template file given as trollstalker logging config,"
" aborting!")
sys.exit()
cparser = ConfigParser()
cparser.read(config_fname)
config = dict(cparser.items(args.config_item, vars=args_dict))
config.update(args_dict)
config.update(
{k: config[k].split(",")
for k in config if "," in config[k]})
config.setdefault("posttroll_port", "0")
try:
log_config = config["stalker_log_config"]
except KeyError:
try:
loglevel = getattr(logging, config.get("loglevel", "DEBUG"))
if loglevel == "":
raise AttributeError
except AttributeError:
loglevel = logging.DEBUG
LOGGER.setLevel(loglevel)
rootlogger = logging.getLogger("")
rootlogger.setLevel(loglevel)
strhndl = logging.StreamHandler()
strhndl.setLevel(loglevel)
log_format = "[%(asctime)s %(levelname)-8s %(name)s] %(message)s"
formatter = logging.Formatter(log_format)
strhndl.setFormatter(formatter)
rootlogger.addHandler(strhndl)
else:
logging.config.fileConfig(log_config)
LOGGER.debug("Logger started")
# Start watching for new files
notifier = FilePublisher(config)
notifier.start()
try:
while True:
time.sleep(6000000)
except KeyboardInterrupt:
LOGGER.info("Interrupting TrollStalker")
finally:
notifier.stop()
# Start gRPC server
server.start()
try:
while True:
time.sleep(_ONE_DAY_IN_SECONDS)
except KeyboardInterrupt:
server.stop(0)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
#config.read(os.path.join(os.path.dirname(__file__), 'config', 'qrag.ini'))
config.read(os.path.join(os.path.dirname(__file__), 'config', 'qrag.ini'))
port = config.get('base', 'port')
parser.add_argument('--port', nargs='?', default=port)
parser.add_argument('--pem_dir', nargs='?')
parser.add_argument('--definition_file',
nargs='?',
default='functions.json')
args = parser.parse_args()
# need to locate the file when script is called from outside it's location dir.
def_file = os.path.join(os.path.dirname(os.path.abspath(__file__)),
args.definition_file)
print(def_file)
logging.info(
'*** Server Configurations Port: {}, Pem_Dir: {}, def_file {} TimeStamp: {} ***'
.format(args.port, args.pem_dir, def_file,
datetime.now().isoformat()))
calc = ExtensionService(def_file)
def _ws_batch(request, context):
"""
Mirrors the input and sends back the same data.
:param request: iterable sequence of bundled rows
:return: the same iterable sequence as received
"""
logging.info('Entering {} TimeStamp: {}'.format(
function_name,
datetime.now().strftime("%H:%M:%S.%f")))
host = socket.gethostname()
ip_addr = socket.gethostbyname(host)
logging.debug('Calling qrag.ini section "{}'.format(q_function_name))
ws_url = config.get(q_function_name, 'ws_url')
token = config.get(q_function_name, 'token')
user_name = config.get(q_function_name, 'username')
batch_size = int(config.get(q_function_name, 'batch_size'))
logging.debug('Batch Size {}'.format(batch_size))
ws_route = config.get(q_function_name, 'ws_route')
logging.info('API Route : {}'.format(ws_route))
# setup Caching
bCache = config.get(q_function_name, 'cache')
logging.debug("Caching is set to {}".format(bCache))
if (bCache.lower() == "true"):
logging.info(
"Caching ****Enabled*** for {}".format(q_function_name))
else:
logging.info(
"Caching ****Disabled**** for {}".format(q_function_name))
md = (('qlik-cache', 'no-store'), )
context.send_initial_metadata(md)
ws_url = ws_url + host + '_' + ip_addr + '_' + user_name + '_'
logging.debug('Full url for ws: {} '.format(ws_url))
ws = create_connection(ws_url)
response_rows = []
outer_counter = 1
inner_counter = 1
request_counter = 1
for request_rows in request:
logging.debug('Printing Request Rows - Request Counter {}'.format(
request_counter))
request_counter += 1
temp = MessageToDict(request_rows)
logging.debug('Temp Message to Dict {}'.format(temp))
test_rows = temp['rows']
logging.debug('Test Rows: {}'.format(test_rows))
request_size = len(test_rows)
logging.debug(
'Bundled Row Number of Rows - {}'.format(request_size))
batches = list(qlist.divide_chunks(test_rows, batch_size))
for i in batches:
payload_t = {"action": ws_route}
logging.debug('PreFix Route Seletection {}'.format(payload_t))
logging.debug(len(batches))
payload_t["data"] = i
logging.debug('Size of payload {}'.format(
pysize.get_size(payload_t)))
logging.debug('Showing Payload: {}'.format(payload_t))
logging.debug('batch number {}'.format(outer_counter))
ws.send(json.dumps(payload_t))
logging.debug('message sent WS')
outer_counter += 1
payload_t.clear()
for j in i:
#logging.debug("Priniting i {}" .format(i))
resp = json.loads(ws.recv())
#logging.debug('Response Type : {}' .format(type(resp)))
logging.debug(
'Counter: {} Payload Size: {} Payload Response: {}'.
format(inner_counter, pysize.get_size(resp), resp))
inner_counter += 1
result = resp['result']
logging.debug('Log Resulst: {}'.format(result))
duals = iter([SSE.Dual(strData=result)])
# logging.debug(duals)
#logging.debug('Printing Duals {}' .format(duals))
# Yield the row data as bundled rows
response_rows.append(SSE.Row(duals=duals))
logging.debug(
'Exiting Inner Loop: Printing j {}'.format(j))
yield SSE.BundledRows(rows=response_rows)
ws.close()
logging.info('Exiting {} TimeStamp: {}'.format(
function_name,
datetime.now().strftime("%H:%M:%S.%f")))
def get_dict_from_old_falcon_cfg(config):
global job_type # TODO: Stop using global for wait_for_file().
job_type = "SGE"
section = 'General'
if config.has_option(section, 'job_type'):
job_type = config.get(section, 'job_type')
pa_concurrent_jobs = 8
if config.has_option(section, 'pa_concurrent_jobs'):
pa_concurrent_jobs = config.getint(section, 'pa_concurrent_jobs')
cns_concurrent_jobs = 8
if config.has_option(section, 'cns_concurrent_jobs'):
cns_concurrent_jobs = config.getint(section, 'cns_concurrent_jobs')
ovlp_concurrent_jobs = 8
if config.has_option(section, 'ovlp_concurrent_jobs'):
ovlp_concurrent_jobs = config.getint(section, 'ovlp_concurrent_jobs')
#appending = False
#if config.has_option(section, 'appending'):
# appending = config.get(section, 'appending')
# if appending == "True":
# appending = True
#openending = False
#if config.has_option(section, 'openending'):
# openending = config.get(section, 'openending')
# if openending == "True":
# openending = True
input_type = "raw"
if config.has_option(section, 'input_type'):
input_type = config.get(section, 'input_type')
overlap_filtering_setting = """--max_diff 1000 --max_cov 1000 --min_cov 2"""
if config.has_option(section, 'overlap_filtering_setting'):
overlap_filtering_setting = config.get(section,
'overlap_filtering_setting')
pa_HPCdaligner_option = """-v -D24 -t16 -e.70 -l1000 -s100"""
if config.has_option(section, 'pa_HPCdaligner_option'):
pa_HPCdaligner_option = config.get(section, 'pa_HPCdaligner_option')
ovlp_HPCdaligner_option = """ -v -D24 -t32 -h60 -e.96 -l500 -s1000"""
if config.has_option(section, 'ovlp_HPCdaligner_option'):
ovlp_HPCdaligner_option = config.get(section,
'ovlp_HPCdaligner_option')
pa_HPCdaligner_option = update_HPCdaligner_option(pa_HPCdaligner_option)
ovlp_HPCdaligner_option = update_HPCdaligner_option(
ovlp_HPCdaligner_option)
pa_DBsplit_option = """ -x500 -s200"""
if config.has_option(section, 'pa_DBsplit_option'):
pa_DBsplit_option = config.get(section, 'pa_DBsplit_option')
dust = False
if config.has_option(section, 'dust'):
dust = config.getboolean(section, 'dust')
pa_DBdust_option = "-w128 -t2.5 -m20"
if config.has_option(section, 'pa_DBdust_option'):
pa_DBdust_option = config.get(section, 'pa_DBdust_option')
dazcon = False
if config.has_option(section, 'dazcon'):
dazcon = config.getboolean(section, 'dazcon')
pa_dazcon_option = "-j 4 -x -l 500"
if config.has_option(section, 'pa_dazcon_option'):
pa_dazcon_option = config.get(section, 'pa_dazcon_option')
ovlp_DBsplit_option = """ -x500 -s200"""
if config.has_option(section, 'ovlp_DBsplit_option'):
ovlp_DBsplit_option = config.get(section, 'ovlp_DBsplit_option')
falcon_sense_option = """ --output_multi --min_idt 0.70 --min_cov 2 --max_n_read 1800 --n_core 6"""
if config.has_option(section, 'falcon_sense_option'):
falcon_sense_option = config.get(section, 'falcon_sense_option')
if 'local_match_count' in falcon_sense_option or 'output_dformat' in falcon_sense_option:
raise Exception(
'Please remove obsolete "--local_match_count_*" or "--output_dformat"'
+ ' from "falcon_sense_option" in your cfg: %s' %
repr(falcon_sense_option))
falcon_sense_skip_contained = False
if config.has_option(section, 'falcon_sense_skip_contained'):
falcon_sense_skip_contained = config.get(
section, 'falcon_sense_skip_contained')
if falcon_sense_skip_contained in ["True", "true", "1"]:
falcon_sense_skip_contained = True
else:
falcon_sense_skip_contained = False
genome_size = 0
if config.has_option(section, 'genome_size'):
genome_size = config.getint(section, 'genome_size')
seed_coverage = 20
if config.has_option(section, 'seed_coverage'):
seed_coverage = config.getfloat(section, 'seed_coverage')
length_cutoff = -1
if config.has_option(section, 'length_cutoff'):
length_cutoff = config.getint(section, 'length_cutoff')
if length_cutoff < 0:
if genome_size < 1:
raise Exception(
'Must specify either length_cutoff>0 or genome_size>0')
length_cutoff_pr = config.getint(section, 'length_cutoff_pr')
input_fofn_fn = config.get(section, 'input_fofn')
# This one depends on length_cutoff_pr for its default.
fc_ovlp_to_graph_option = ''
if config.has_option(section, 'fc_ovlp_to_graph_option'):
fc_ovlp_to_graph_option = config.get(section,
'fc_ovlp_to_graph_option')
if '--min_len' not in fc_ovlp_to_graph_option:
fc_ovlp_to_graph_option += ' --min_len %d' % length_cutoff_pr
bestn = 12
if config.has_option(section, 'bestn'):
bestn = config.getint(section, 'bestn')
if config.has_option(section, 'target'):
target = config.get(section, 'target')
if target not in ["overlapping", "pre-assembly", "assembly"]:
msg = """ Target has to be "overlapping", "pre-assembly" or "assembly" in this verison. You have an unknown target %s in the configuration file. """ % target
raise Exception(msg)
else:
logger.info(""" No target specified, assuming "assembly" as target """)
target = "assembly"
if config.has_option(section, 'stop_all_jobs_on_failure'):
stop_all_jobs_on_failure = config.getboolean(
section, 'stop_all_jobs_on_failure')
else:
# Good default. Rarely needed, since we already stop early if *all* tasks fail
# in a given refresh.
stop_all_jobs_on_failure = False
if config.has_option(section, 'use_tmpdir'):
use_tmpdir = config.getboolean(section, 'use_tmpdir')
else:
use_tmpdir = False
TEXT_FILE_BUSY = 'avoid_text_file_busy'
if config.has_option(section, TEXT_FILE_BUSY):
bash.BUG_avoid_Text_file_busy = config.getboolean(
section, TEXT_FILE_BUSY)
hgap_config = {#"input_fofn_fn" : input_fofn_fn, # deprecated
"input_fofn" : input_fofn_fn,
"target" : target,
"job_type" : job_type,
"input_type": input_type,
#"openending": openending,
"pa_concurrent_jobs" : pa_concurrent_jobs,
"ovlp_concurrent_jobs" : ovlp_concurrent_jobs,
"cns_concurrent_jobs" : cns_concurrent_jobs,
"overlap_filtering_setting": overlap_filtering_setting,
"genome_size" : genome_size,
"seed_coverage" : seed_coverage,
"length_cutoff" : length_cutoff,
"length_cutoff_pr" : length_cutoff_pr,
"sge_option_da": config.get(section, 'sge_option_da'),
"sge_option_la": config.get(section, 'sge_option_la'),
"sge_option_pda": config.get(section, 'sge_option_pda'),
"sge_option_pla": config.get(section, 'sge_option_pla'),
"sge_option_fc": config.get(section, 'sge_option_fc'),
"sge_option_cns": config.get(section, 'sge_option_cns'),
"pa_HPCdaligner_option": pa_HPCdaligner_option,
"ovlp_HPCdaligner_option": ovlp_HPCdaligner_option,
"pa_DBsplit_option": pa_DBsplit_option,
"dust": dust,
"pa_DBdust_option": pa_DBdust_option,
"dazcon": dazcon,
"pa_dazcon_option": pa_dazcon_option,
"ovlp_DBsplit_option": ovlp_DBsplit_option,
"fc_ovlp_to_graph_option": fc_ovlp_to_graph_option,
"falcon_sense_option": falcon_sense_option,
"falcon_sense_skip_contained": falcon_sense_skip_contained,
"stop_all_jobs_on_failure": stop_all_jobs_on_failure,
"use_tmpdir": use_tmpdir,
TEXT_FILE_BUSY: bash.BUG_avoid_Text_file_busy,
}
provided = dict(config.items(section))
unused = set(provided) - set(k.lower() for k in hgap_config)
if unused:
import warnings
warnings.warn("Unexpected keys in input config: %s" % repr(unused))
hgap_config["install_prefix"] = sys.prefix
return hgap_config
def _rest_template(request, context):
"""
Rest using template
"""
logging.info('Entering {} TimeStamp: {}'.format(
function_name,
datetime.now().strftime("%H:%M:%S.%f")))
url = config.get(q_function_name, 'url')
logging.debug("Rest Url is set to {}".format(url))
query_parameter = config.get(q_function_name, 'query_parameter')
logging.debug("Rest Url is set to {}".format(query_parameter))
bCache = config.get(q_function_name, 'cache')
logging.debug("Caching is set to {}".format(bCache))
if (bCache.lower() == "true"):
logging.info(
"Caching ****Enabled*** for {}".format(q_function_name))
else:
logging.info(
"Caching ****Disabled**** for {}".format(q_function_name))
md = (('qlik-cache', 'no-store'), )
context.send_initial_metadata(md)
response_rows = []
request_counter = 1
for request_rows in request:
logging.debug('Printing Request Rows - Request Counter {}'.format(
request_counter))
request_counter = request_counter + 1
for row in request_rows.rows:
# Retrieve string value of parameter and append to the params variable
# Length of param is 1 since one column is received, the [0] collects the first value in the list
param = [d.strData for d in row.duals]
print(param)
print(type(param))
# Join with current timedate stamp
if (len(param) == 0):
logging.info(
'Exiting {} TimeStamp: {} due to Data being Empty '.
format(function_name,
datetime.now().strftime("%H:%M:%S.%f")))
else:
qp0 = param[0]
qp1 = param[1]
logging.debug('Showing Payload: {}'.format(qp0))
logging.debug('Showing Payload: {}'.format(qp1))
logging.debug(
'Showing Payload: {}'.format(query_parameter))
parameterized_url = url + qp0 + '?' + query_parameter + '=' + qp1 + '&'
logging.debug(
'Showing Payload: {}'.format(parameterized_url))
resp = requests.get(parameterized_url)
logging.debug('Type: {}'.format(type(resp)))
logging.debug('Show Payload Response as Text: {}'.format(
resp.json()))
result = resp.json()
result = result.get('ratings')
#result = result.strip()
logging.debug('Show Result: {}'.format(result))
# Create an iterable of dual with the result
for i in result:
retData = str(i)
print(retData)
print(type(retData))
duals = iter([SSE.Dual(strData=retData)])
response_rows.append(SSE.Row(duals=duals))
# Yield the row data as bundled rows
yield SSE.BundledRows(rows=response_rows)
logging.info('Exiting {} TimeStamp: {}'.format(
function_name,
datetime.now().strftime("%H:%M:%S.%f")))
raw_list = None
loc_raw = args.raw_repo
loc_output = args.loc_out
config = ConfigParser.RawConfigParser()
config.read(
os.path.join(os.path.dirname(os.path.realpath(sys.argv[0])),
'moff_setting.properties'))
df = pd.read_csv(file_name, sep="\t")
## add same safety checks len > 1
## Flag for pride pipeline, or to set from second to minute as input rt time scale
moff_pride_flag = 0
if moff.check_ps_input_data(
df.columns.tolist(),
ast.literal_eval(config.get('moFF', 'moffpride_format'))) == 1:
# if it is a moff_pride data I do not check aany other requirement
log.critical('moffPride input detected')
moff_pride_flag = 1
else:
if not 'matched' in df.columns:
# check if it is a PS file ,
list_name = df.columns.values.tolist()
# get the lists of PS defaultcolumns from properties file
list = ast.literal_eval(
config.get('moFF', 'ps_default_export_v1'))
# here it controls if the input file is a PS export; if yes it maps the input in right moFF name
if moff.check_ps_input_data(list_name, list) == 1:
# map the columns name according to moFF input requirements
if args.peptide_summary != 1:
data_ms2, list_name = moff.map_ps2moff(
def main():
parser = argparse.ArgumentParser(description='FIX Gateway')
parser.add_argument('--debug',
action='store_true',
help='Run in debug mode')
parser.add_argument('--config-file',
type=argparse.FileType('r'),
help='Alternate configuration file')
parser.add_argument('--log-config',
type=argparse.FileType('w'),
help='Alternate logger configuration file')
args, unknown_args = parser.parse_known_args()
logging.config.fileConfig(logconfig_file)
log = logging.getLogger()
if args.debug:
log.setLevel(logging.DEBUG)
log.info("Starting FIX Gateway")
config = configparser.ConfigParser()
# To kepp configparser from making everythign lowercase
config.optionxform = str
config.read(config_file)
try:
database.init(config)
except Exception as e:
log.error("Database failure, Exiting")
print(e)
raise
return # we don't want to run with a screwed up database
log.info("Setting Initial Values")
try:
for item in config.items("initial"):
database.write(item[0], item[1])
except Exception as e:
log.error(
"Problem setting initial values from configuration - {0}".format(
e))
# TODO: Add a hook here for post database creation code
# TODO: Need to do some more thorough error checking here
# run through the plugin_list dict and find all the plugins that are
# configured to be loaded and load them.
for each in config:
if each[:5] == "conn_":
if config.getboolean(each, "load"):
module = config.get(each, "module")
try:
load_plugin(each[5:], module, config)
except Exception as e:
logging.critical("Unable to load module - " + module +
": " + str(e))
# TODO add a hook here for pre module run code
for each in plugins:
plugins[each].run()
iteration = 0
while True:
try:
job = plugin.jobQueue.get(timeout=1.0)
if job == "QUIT":
break
except KeyboardInterrupt:
log.info("Termination from keybaord received")
break
except queue.Empty:
pass
iteration += 1
# Every four times through the loop we do some stuff
if iteration % 4 == 0:
# check how many plugins are running and exit if zero
running_count = 0
for each in plugins:
if plugins[each].is_running():
running_count += 1
if running_count == 0:
log.info("No plugins running, quitting")
break
for each in plugins:
plugins[each].stop()
log.info("FIX Gateway Exiting Normally")
def create_app(main_module=False):
app = Flask(__name__, static_url_path='/static', static_folder='static')
babel = Babel(app)
app.config['BABEL_TRANSLATION_DIRECTORIES'] = os.path.abspath(
'caipirinha/i18n/locales')
logging.config.fileConfig('logging_config.ini')
app.secret_key = 'l3m0n4d1'
# CORS
CORS(app, resources={r"/*": {"origins": "*"}})
# Swagger
swaggerui_blueprint = get_swaggerui_blueprint(
'/api/docs',
'/static/swagger.yaml',
config={ # Swagger UI config overrides
'app_name': "Lemonade Caipirinha"
},
# oauth_config={ # OAuth config. See https://github.com/swagger-api/swagger-ui#oauth2-configuration .
# 'clientId': "your-client-id",
# 'clientSecret': "your-client-secret-if-required",
# 'realm': "your-realms",
# 'appName': "your-app-name",
# 'scopeSeparator': " ",
# 'additionalQueryStringParams': {'test': "hello"}
# }
)
app.register_blueprint(swaggerui_blueprint)
# Error handling
app.register_error_handler(404, handle_exception)
# API
api = Api(app)
mappings = {
'/dashboards': [DashboardListApi, 'dashboardList'],
'/public/dashboard/<h>': [PublicDashboardApi, 'publicDashboard'],
'/dashboards/<int:dashboard_id>':
[DashboardDetailApi, 'dashboardDetail'],
'/visualizations/<int:job_id>/<task_id>/<int:vis_id>':
[VisualizationDetailApi, 'visualizationDetail'],
'/visualizations/<int:job_id>/<task_id>':
[VisualizationDetailApi, 'visualizationDetailOld'],
'/visualizations/<int:job_id>/<task_id>':
[VisualizationDetailApi, 'visualizationDetailOld'],
'/public/visualization/<int:job_id>/<task_id>/<int:vis_id>':
[PublicVisualizationApi, 'publicVisualization'],
'/visualizations': [VisualizationListApi, 'visualizationList'],
'/texts': [TextListApi, 'textList'],
'/texts/<int:text_id>': [TextDetailApi, 'textDetail'],
}
for path, view in list(mappings.items()):
api.add_resource(view[0], path, endpoint=view[1])
config_file = os.environ.get('CAIPIRINHA_CONFIG')
os.chdir(os.environ.get('CAIPIRINHA_HOME', '.'))
logger = logging.getLogger(__name__)
migrate = Migrate(app, db)
@babel.localeselector
def get_locale():
user = getattr(flask_g, 'user', None)
if user is not None and user.locale:
return user.locale
else:
return request.args.get(
'lang', request.accept_languages.best_match(['en', 'pt',
'es']))
sqlalchemy_utils.i18n.get_locale = get_locale
if config_file:
with open(config_file) as f:
config = yaml.load(f, Loader=yaml.FullLoader)['caipirinha']
app.config["RESTFUL_JSON"] = {"cls": app.json_encoder}
server_config = config.get('servers', {})
app.config['SQLALCHEMY_DATABASE_URI'] = server_config.get(
'database_url')
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
engine_config = config.get('config', {})
if engine_config:
final_config = {'pool_pre_ping': True}
if 'mysql://' in app.config['SQLALCHEMY_DATABASE_URI']:
if 'SQLALCHEMY_POOL_SIZE' in engine_config:
final_config['pool_size'] = engine_config[
'SQLALCHEMY_POOL_SIZE']
if 'SQLALCHEMY_POOL_RECYCLE' in engine_config:
final_config['pool_recycle'] = engine_config[
'SQLALCHEMY_POOL_RECYCLE']
app.config['SQLALCHEMY_ENGINE_OPTIONS'] = final_config
app.config['CAIPIRINHA_CONFIG'] = config
db.init_app(app)
port = int(config.get('port', 5000))
logger.debug('Running in %s mode', config.get('environment'))
if main_module:
if config.get('environment', 'dev') == 'dev':
app.run(debug=True, port=port)
else:
eventlet.wsgi.server(eventlet.listen(('', port)), app)
else:
return app
else:
logger.error('Please, set CAIPIRINHA_CONFIG environment variable')
exit(1)
import os
import logging.config
from holygrail import optHisNavHihestWinLowest
from configparser import ConfigParser
from holygrail.dealutils import *
from holygrail.btinfo import Btinfo
logging.config.fileConfig(
'/Users/lamter/workspace/SlaveO/holygrail/tmp/logging.ini')
configIni = '/Users/lamter/workspace/SlaveO/holygrail/tmp/oscillationDonchian_opt.ini'
config = ConfigParser()
config.read(configIni)
path = config.get('优化', 'path')
db = getDB(config)
group, className = '测试ATR通道震荡', 'OscillationAtrChannelStrategy'
periodList = ['{}BM'.format(i) for i in range(1, 13)]
histNavRangeList = list(range(1, 101))
# histNavRangeList = list(range(10, 25))
btinfoCol = db[config.get('回测相关数MongoDB据库', 'btinfo')]
d = btinfoCol.find_one({'group': group, 'className': className}, {'_id': 0})
btInfo = Btinfo(**d)
usList = btInfo.underlyingSymbols[:]
usList = ['rb']
import logging, time, sys, logging.config, autologging, os
import ConfigParser
# All the logging bootstrapping is done here
env = os.getenv("env", "sandbox")
config_file = "/etc/kafka-backpressure-monitor/settings_{}.ini".format(env)
config = ConfigParser.ConfigParser()
config.read(config_file)
log_level_key = 'consumer_log_level'
consumer_log_level = config.get('log', log_level_key)
consumer_log_level = os.getenv(log_level_key, consumer_log_level).upper()
# or we can just use getattr() instead as commented below
if consumer_log_level == 'CRITICAL':
log_level = logging.CRITICAL
elif consumer_log_level == 'ERROR':
log_level = logging.ERROR
elif consumer_log_level == 'WARNING':
log_level = logging.WARNING
elif consumer_log_level == 'INFO':
log_level = logging.INFO
else:
log_level = logging.DEBUG
out_type = config.get('log', 'out_type')
log_location = config.get('log', 'location')
log_file = log_location + "kafka_backpressure_monitor.log"
# Handles all the loggers to use across the project
gen_logger = logging.getLogger("kafka.backpressure")
#log_level = getattr(gen_logger, consumer_log_level)
gen_logger.setLevel(log_level)
formatter = logging.Formatter('%(asctime)s:%(name)s:%(levelname)s:process %(process)d:process name %(processName)s:%(funcName)s():line %(lineno)d:%(message)s')
def _transition_queued_actions_to_pending(self):
_logger.info('transitioning queued actions to pending')
action_service = self._action_service
engine_service = self._engine_service
datastore_service = self._datastore_service
assert isinstance(action_service, ActionService)
assert isinstance(engine_service, EngineService)
assert isinstance(datastore_service, DatastoreService)
queued_actions = action_service.find_actions(
states=[ActionState.QUEUED])
for action in queued_actions:
try:
datastore = datastore_service.get_datastore(
action.data.datastore_id, raise_when_missing=False)
if not datastore or datastore.data.state != DatastoreState.ACTIVE:
continue
states = [
ActionState.PENDING, ActionState.RUNNING,
ActionState.FINISHING
]
action_count = self._action_service.find_action_count(
datastore.id, states)
if action_count >= datastore.data.concurrency:
_logger.info(
'datastore (id=%s) has reached max concurrency' %
datastore.id)
continue
# conditionally updating queued actions as pending allows multiple concurrent engine workers if needed
action_service.update_action_state(
action=action,
state=ActionState.PENDING,
error_message=action.data.error_message,
conditional=lambda a: a.data.state == ActionState.QUEUED)
engine = engine_service.get_engine_by_name(
action.data.engine_name)
# our best granualrity of a user_id to identifu who is running this workflow's action.
datastore_user_id = datastore.data.user_id if hasattr(
datastore.data, 'user_id') else 'anonymous'
if self.dart_config['dart'].get('use_local_engines'):
config = self.dart_config['engines'][engine.data.name]
engine_instance = locate(
config['path'])(**config.get('options', {}))
self._launch_in_memory_engine(engine, engine_instance,
action, datastore_user_id)
# empty string allows differentiation from null, yet is still falsey
action_service.update_action_batch_job_id(action, '')
elif engine.data.ecs_task_definition_arn:
batch_job_id = self._try_run_task(action,
datastore_user_id)
if batch_job_id:
action_service.update_action_batch_job_id(
action, batch_job_id)
else:
# no batch job-id means there isn't enough capacity at the moment, so try again later
action_service.update_action_state(
action, ActionState.QUEUED,
action.data.error_message)
else:
msg = 'engine %s has no ecs_task_definition and local engines are not allowed'
raise Exception(msg % engine.data.name)
except DartConditionalUpdateFailedException:
# another engine worker picked it up
continue
except Exception as e:
error_message = e.message + '\n\n\n' + traceback.format_exc()
_logger.error(
'error transitioning action (id=%s) to PENDING: %s' %
(action.id, error_message))
finally:
db.session.rollback()
data_dir.mkdir(parents=True, exist_ok=True)
return config_dict
config = Config(env_prefix="A3M", attrs=CONFIG_MAPPING)
config.read_defaults(StringIO(CONFIG_DEFAULTS))
config.read_files(["/etc/a3m/a3m.cfg"])
config.read_dict(_get_data_dir_defaults(config))
# Django
DATABASES = {
"default": {
"ENGINE": config.get("db_engine"),
"NAME": config.get("db_name"),
"USER": config.get("db_user"),
"PASSWORD": config.get("db_password"),
"HOST": config.get("db_host"),
"PORT": config.get("db_port"),
"CONN_MAX_AGE": 3600,
"OPTIONS": {"timeout": 5},
}
}
MIDDLEWARE_CLASSES = ()
TEMPLATES = [{"BACKEND": "django.template.backends.django.DjangoTemplates"}]
INSTALLED_APPS = ("a3m.main", "a3m.fpr")
import logging.config
import os
from config42 import ConfigManager
from instabot_py.default_config import DEFAULT_CONFIG
env_config = ConfigManager(prefix="INSTABOT")
logging.basicConfig(
level=logging.DEBUG if env_config.get("debug") else logging.INFO)
LOGGER = logging.getLogger(__name__)
config = ConfigManager()
config.set_many(DEFAULT_CONFIG)
config.set_many(env_config.as_dict())
config_file = config.get("config.file")
config_etcd = config.get("config.etcd")
if config_file:
if config_file.startswith("/"):
config_path = config_file
else:
cwd = os.getcwd()
config_path = cwd + "/" + config_file
config.set_many(
ConfigManager(path=config_path.replace('//', '/')).as_dict())
LOGGER.info("Setting configuration from {} : OK".format(config_file))
if config_etcd:
if not config_etcd.get("keyspace"):
raise Exception("etcd Keyspace is mandatory")
def getFFmpegWait():
if config.has_option('Server', 'ffmpeg_wait'):
return max(int(float(config.get('Server', 'ffmpeg_wait'))), 1)
else:
return 0
"""
log = logging.getLogger(__name__)
config = ConfigParser.RawConfigParser()
try:
log_path = config.get('LOG_Path', 'log_path')
except ConfigParser.ParsingError, err:
print 'Could not parse:', err
mkdir_log(log_path)
fullPath = findLiotaConfigFullPath().get_liota_fullpath()
if fullPath != '':
try:
if config.read(fullPath) != []:
# now use json file for logging settings
try:
log_cfg = config.get('LOG_CFG', 'json_path')
except ConfigParser.ParsingError, err:
print 'Could not parse:', err
else:
raise IOError('Cannot open configuration file ' + fullPath)
except IOError, err:
print 'Could not open:', err
if os.path.exists(log_cfg):
with open(log_cfg, 'rt') as f:
config = json.load(f)
logging.config.dictConfig(config)
log.info('created logger with ' + log_cfg)
else:
# missing logging.json file
logging.basicConfig(level=default_level)
log.warn(
def get_config(parse_args=True, cfg_path=None, options=None):
if parse_args:
options, _ = get_parsed_args()
# General config
agentConfig = {
'check_freq': DEFAULT_CHECK_FREQUENCY,
'dogstatsd_port': 8125,
'dogstatsd_target': 'http://localhost:17123',
'graphite_listen_port': None,
'hostname': None,
'listen_port': None,
'tags': None,
'use_ec2_instance_id': False, # DEPRECATED
'version': get_version(),
'watchdog': True,
'additional_checksd': '/etc/dd-agent/checks.d/',
'bind_host': get_default_bind_host(),
'statsd_metric_namespace': None,
'utf8_decoding': False
}
if Platform.is_mac():
agentConfig['additional_checksd'] = '/opt/datadog-agent/etc/checks.d'
# Config handling
try:
# Find the right config file
path = os.path.realpath(__file__)
path = os.path.dirname(path)
config_path = get_config_path(cfg_path, os_name=get_os())
config = ConfigParser.ConfigParser()
config.readfp(skip_leading_wsp(open(config_path)))
# bulk import
for option in config.options('Main'):
agentConfig[option] = config.get('Main', option)
# Store developer mode setting in the agentConfig
if config.has_option('Main', 'developer_mode'):
agentConfig['developer_mode'] = _is_affirmative(
config.get('Main', 'developer_mode'))
# Allow an override with the --profile option
if options is not None and options.profile:
agentConfig['developer_mode'] = True
#
# Core config
#
# FIXME unnecessarily complex
agentConfig['use_forwarder'] = False
if options is not None and options.use_forwarder:
listen_port = 17123
if config.has_option('Main', 'listen_port'):
listen_port = int(config.get('Main', 'listen_port'))
agentConfig['dd_url'] = "http://" + agentConfig[
'bind_host'] + ":" + str(listen_port)
agentConfig['use_forwarder'] = True
elif options is not None and not options.disable_dd and options.dd_url:
agentConfig['dd_url'] = options.dd_url
else:
agentConfig['dd_url'] = config.get('Main', 'dd_url')
if agentConfig['dd_url'].endswith('/'):
agentConfig['dd_url'] = agentConfig['dd_url'][:-1]
# Extra checks.d path
# the linux directory is set by default
if config.has_option('Main', 'additional_checksd'):
agentConfig['additional_checksd'] = config.get(
'Main', 'additional_checksd')
elif get_os() == 'windows':
# default windows location
common_path = _windows_commondata_path()
agentConfig['additional_checksd'] = os.path.join(
common_path, 'Datadog', 'checks.d')
if config.has_option('Main', 'use_dogstatsd'):
agentConfig['use_dogstatsd'] = config.get(
'Main', 'use_dogstatsd').lower() in ("yes", "true")
else:
agentConfig['use_dogstatsd'] = True
# Concerns only Windows
if config.has_option('Main', 'use_web_info_page'):
agentConfig['use_web_info_page'] = config.get(
'Main', 'use_web_info_page').lower() in ("yes", "true")
else:
agentConfig['use_web_info_page'] = True
# Which API key to use
agentConfig['api_key'] = config.get('Main', 'api_key')
# local traffic only? Default to no
agentConfig['non_local_traffic'] = False
if config.has_option('Main', 'non_local_traffic'):
agentConfig['non_local_traffic'] = config.get(
'Main', 'non_local_traffic').lower() in ("yes", "true")
# DEPRECATED
if config.has_option('Main', 'use_ec2_instance_id'):
use_ec2_instance_id = config.get('Main', 'use_ec2_instance_id')
# translate yes into True, the rest into False
agentConfig['use_ec2_instance_id'] = (
use_ec2_instance_id.lower() == 'yes')
if config.has_option('Main', 'check_freq'):
try:
agentConfig['check_freq'] = int(
config.get('Main', 'check_freq'))
except Exception:
pass
# Custom histogram aggregate/percentile metrics
if config.has_option('Main', 'histogram_aggregates'):
agentConfig['histogram_aggregates'] = get_histogram_aggregates(
config.get('Main', 'histogram_aggregates'))
if config.has_option('Main', 'histogram_percentiles'):
agentConfig['histogram_percentiles'] = get_histogram_percentiles(
config.get('Main', 'histogram_percentiles'))
# Disable Watchdog (optionally)
if config.has_option('Main', 'watchdog'):
if config.get('Main', 'watchdog').lower() in ('no', 'false'):
agentConfig['watchdog'] = False
# Optional graphite listener
if config.has_option('Main', 'graphite_listen_port'):
agentConfig['graphite_listen_port'] = \
int(config.get('Main', 'graphite_listen_port'))
else:
agentConfig['graphite_listen_port'] = None
# Dogstatsd config
dogstatsd_defaults = {
'dogstatsd_port': 8125,
'dogstatsd_target':
'http://' + agentConfig['bind_host'] + ':17123',
}
for key, value in dogstatsd_defaults.iteritems():
if config.has_option('Main', key):
agentConfig[key] = config.get('Main', key)
else:
agentConfig[key] = value
# Create app:xxx tags based on monitored apps
agentConfig['create_dd_check_tags'] = config.has_option('Main', 'create_dd_check_tags') and \
_is_affirmative(config.get('Main', 'create_dd_check_tags'))
# Forwarding to external statsd server
if config.has_option('Main', 'statsd_forward_host'):
agentConfig['statsd_forward_host'] = config.get(
'Main', 'statsd_forward_host')
if config.has_option('Main', 'statsd_forward_port'):
agentConfig['statsd_forward_port'] = int(
config.get('Main', 'statsd_forward_port'))
# optionally send dogstatsd data directly to the agent.
if config.has_option('Main', 'dogstatsd_use_ddurl'):
if _is_affirmative(config.get('Main', 'dogstatsd_use_ddurl')):
agentConfig['dogstatsd_target'] = agentConfig['dd_url']
# Optional config
# FIXME not the prettiest code ever...
if config.has_option('Main', 'use_mount'):
agentConfig['use_mount'] = _is_affirmative(
config.get('Main', 'use_mount'))
if options is not None and options.autorestart:
agentConfig['autorestart'] = True
elif config.has_option('Main', 'autorestart'):
agentConfig['autorestart'] = _is_affirmative(
config.get('Main', 'autorestart'))
if config.has_option('Main', 'check_timings'):
agentConfig['check_timings'] = _is_affirmative(
config.get('Main', 'check_timings'))
if config.has_option('Main', 'exclude_process_args'):
agentConfig['exclude_process_args'] = _is_affirmative(
config.get('Main', 'exclude_process_args'))
try:
filter_device_re = config.get('Main', 'device_blacklist_re')
agentConfig['device_blacklist_re'] = re.compile(filter_device_re)
except ConfigParser.NoOptionError:
pass
if config.has_option('datadog', 'ddforwarder_log'):
agentConfig['has_datadog'] = True
# Dogstream config
if config.has_option("Main", "dogstream_log"):
# Older version, single log support
log_path = config.get("Main", "dogstream_log")
if config.has_option("Main", "dogstream_line_parser"):
agentConfig["dogstreams"] = ':'.join(
[log_path,
config.get("Main", "dogstream_line_parser")])
else:
agentConfig["dogstreams"] = log_path
elif config.has_option("Main", "dogstreams"):
agentConfig["dogstreams"] = config.get("Main", "dogstreams")
if config.has_option("Main", "nagios_perf_cfg"):
agentConfig["nagios_perf_cfg"] = config.get(
"Main", "nagios_perf_cfg")
if config.has_option("Main", "use_curl_http_client"):
agentConfig["use_curl_http_client"] = _is_affirmative(
config.get("Main", "use_curl_http_client"))
else:
# Default to False as there are some issues with the curl client and ELB
agentConfig["use_curl_http_client"] = False
if config.has_section('WMI'):
agentConfig['WMI'] = {}
for key, value in config.items('WMI'):
agentConfig['WMI'][key] = value
if (config.has_option("Main", "limit_memory_consumption") and
config.get("Main", "limit_memory_consumption") is not None):
agentConfig["limit_memory_consumption"] = int(
config.get("Main", "limit_memory_consumption"))
else:
agentConfig["limit_memory_consumption"] = None
if config.has_option("Main", "skip_ssl_validation"):
agentConfig["skip_ssl_validation"] = _is_affirmative(
config.get("Main", "skip_ssl_validation"))
agentConfig["collect_instance_metadata"] = True
if config.has_option("Main", "collect_instance_metadata"):
agentConfig["collect_instance_metadata"] = _is_affirmative(
config.get("Main", "collect_instance_metadata"))
agentConfig["proxy_forbid_method_switch"] = False
if config.has_option("Main", "proxy_forbid_method_switch"):
agentConfig["proxy_forbid_method_switch"] = _is_affirmative(
config.get("Main", "proxy_forbid_method_switch"))
agentConfig["collect_ec2_tags"] = False
if config.has_option("Main", "collect_ec2_tags"):
agentConfig["collect_ec2_tags"] = _is_affirmative(
config.get("Main", "collect_ec2_tags"))
agentConfig["utf8_decoding"] = False
if config.has_option("Main", "utf8_decoding"):
agentConfig["utf8_decoding"] = _is_affirmative(
config.get("Main", "utf8_decoding"))
agentConfig["gce_updated_hostname"] = False
if config.has_option("Main", "gce_updated_hostname"):
agentConfig["gce_updated_hostname"] = _is_affirmative(
config.get("Main", "gce_updated_hostname"))
except ConfigParser.NoSectionError, e:
sys.stderr.write('Config file not found or incorrectly formatted.\n')
sys.exit(2)
def get_server(name, default=None):
if config.has_option('Server', name):
return config.get('Server', name)
else:
return default
def config_get_safe(config, section, key, default=None):
try:
return config.get(section, key)
except configparser.NoOptionError:
return default
logger (Logger): the object for logging
"""
import ConfigParser
import datetime
import logging.config
import os
from bs4 import BeautifulSoup
import requests
config = ConfigParser.RawConfigParser()
config.read('/etc/calfresh/calfresh.conf')
temp_dir = config.get('filepaths', 'temp')
data_dir = config.get('filepaths', 'data')
logging.config.fileConfig(config.get('filepaths', 'config'))
logger = logging.getLogger('web_crawler')
class WebCrawler(object):
"""The WebCrawler gets today's and yesterday's html files for a given url
and uses the PageParser to identify new and updated files
Args:
table (str): the table we're currently working on
url (str): the url for the table's data
Returns:
def get_config(parse_args=True, cfg_path=None, options=None):
if parse_args:
options, _ = get_parsed_args()
# General config
agentConfig = {
'check_freq': DEFAULT_CHECK_FREQUENCY,
'dogstatsd_interval': DEFAULT_STATSD_FREQUENCY,
'dogstatsd_normalize': 'yes',
'dogstatsd_port': 8125,
'dogstatsd_target': 'http://localhost:17123',
'graphite_listen_port': None,
'hostname': None,
'listen_port': None,
'tags': None,
'use_ec2_instance_id': False, # DEPRECATED
'version': get_version(),
'watchdog': True,
'additional_checksd': '/etc/dd-agent/checks.d/',
}
dogstatsd_interval = DEFAULT_STATSD_FREQUENCY
# Config handling
try:
# Find the right config file
path = os.path.realpath(__file__)
path = os.path.dirname(path)
config_path = get_config_path(cfg_path, os_name=get_os())
config = ConfigParser.ConfigParser()
config.readfp(skip_leading_wsp(open(config_path)))
# bulk import
for option in config.options('Main'):
agentConfig[option] = config.get('Main', option)
#
# Core config
#
# FIXME unnecessarily complex
if config.has_option('Main', 'use_dd'):
agentConfig['use_dd'] = config.get('Main',
'use_dd').lower() in ("yes",
"true")
else:
agentConfig['use_dd'] = True
agentConfig['use_forwarder'] = False
if options is not None and options.use_forwarder:
listen_port = 17123
if config.has_option('Main', 'listen_port'):
listen_port = int(config.get('Main', 'listen_port'))
agentConfig['dd_url'] = "http://localhost:" + str(listen_port)
agentConfig['use_forwarder'] = True
elif options is not None and not options.disable_dd and options.dd_url:
agentConfig['dd_url'] = options.dd_url
else:
agentConfig['dd_url'] = config.get('Main', 'dd_url')
if agentConfig['dd_url'].endswith('/'):
agentConfig['dd_url'] = agentConfig['dd_url'][:-1]
# Extra checks.d path
# the linux directory is set by default
if config.has_option('Main', 'additional_checksd'):
agentConfig['additional_checksd'] = config.get(
'Main', 'additional_checksd')
elif get_os() == 'windows':
# default windows location
common_path = _windows_commondata_path()
agentConfig['additional_checksd'] = os.path.join(
common_path, 'Datadog', 'checks.d')
# Whether also to send to Pup
if config.has_option('Main', 'use_pup'):
agentConfig['use_pup'] = config.get('Main',
'use_pup').lower() in ("yes",
"true")
else:
agentConfig['use_pup'] = True
# Concerns only Windows
if config.has_option('Main', 'use_web_info_page'):
agentConfig['use_web_info_page'] = config.get(
'Main', 'use_web_info_page').lower() in ("yes", "true")
else:
agentConfig['use_web_info_page'] = True
if agentConfig['use_pup'] or agentConfig['use_web_info_page']:
if config.has_option('Main', 'pup_url'):
agentConfig['pup_url'] = config.get('Main', 'pup_url')
else:
agentConfig['pup_url'] = 'http://localhost:17125'
if config.has_option('Main', 'pup_port'):
agentConfig['pup_port'] = int(config.get('Main', 'pup_port'))
# Increases the frequency of statsd metrics when only sending to Pup
if not agentConfig['use_dd'] and agentConfig['use_pup']:
dogstatsd_interval = PUP_STATSD_FREQUENCY
if not agentConfig['use_dd'] and not agentConfig['use_pup']:
sys.stderr.write(
"Please specify at least one endpoint to send metrics to. This can be done in datadog.conf."
)
exit(2)
# Which API key to use
agentConfig['api_key'] = config.get('Main', 'api_key')
# local traffic only? Default to no
agentConfig['non_local_traffic'] = False
if config.has_option('Main', 'non_local_traffic'):
agentConfig['non_local_traffic'] = config.get(
'Main', 'non_local_traffic').lower() in ("yes", "true")
# DEPRECATED
if config.has_option('Main', 'use_ec2_instance_id'):
use_ec2_instance_id = config.get('Main', 'use_ec2_instance_id')
# translate yes into True, the rest into False
agentConfig['use_ec2_instance_id'] = (
use_ec2_instance_id.lower() == 'yes')
if config.has_option('Main', 'check_freq'):
try:
agentConfig['check_freq'] = int(
config.get('Main', 'check_freq'))
except Exception:
pass
# Disable Watchdog (optionally)
if config.has_option('Main', 'watchdog'):
if config.get('Main', 'watchdog').lower() in ('no', 'false'):
agentConfig['watchdog'] = False
# Optional graphite listener
if config.has_option('Main', 'graphite_listen_port'):
agentConfig['graphite_listen_port'] = \
int(config.get('Main', 'graphite_listen_port'))
else:
agentConfig['graphite_listen_port'] = None
# Dogstatsd config
dogstatsd_defaults = {
'dogstatsd_port': 8125,
'dogstatsd_target': 'http://localhost:17123',
'dogstatsd_interval': dogstatsd_interval,
'dogstatsd_normalize': 'yes',
}
for key, value in dogstatsd_defaults.iteritems():
if config.has_option('Main', key):
agentConfig[key] = config.get('Main', key)
else:
agentConfig[key] = value
#Forwarding to external statsd server
if config.has_option('Main', 'statsd_forward_host'):
agentConfig['statsd_forward_host'] = config.get(
'Main', 'statsd_forward_host')
if config.has_option('Main', 'statsd_forward_port'):
agentConfig['statsd_forward_port'] = int(
config.get('Main', 'statsd_forward_port'))
# normalize 'yes'/'no' to boolean
dogstatsd_defaults['dogstatsd_normalize'] = _is_affirmative(
dogstatsd_defaults['dogstatsd_normalize'])
# optionally send dogstatsd data directly to the agent.
if config.has_option('Main', 'dogstatsd_use_ddurl'):
use_ddurl = _is_affirmative(
config.get('Main', 'dogstatsd_use_ddurl'))
if use_ddurl:
agentConfig['dogstatsd_target'] = agentConfig['dd_url']
# Optional config
# FIXME not the prettiest code ever...
if config.has_option('Main', 'use_mount'):
agentConfig['use_mount'] = _is_affirmative(
config.get('Main', 'use_mount'))
if config.has_option('Main', 'autorestart'):
agentConfig['autorestart'] = _is_affirmative(
config.get('Main', 'autorestart'))
try:
filter_device_re = config.get('Main', 'device_blacklist_re')
agentConfig['device_blacklist_re'] = re.compile(filter_device_re)
except ConfigParser.NoOptionError:
pass
if config.has_option('datadog', 'ddforwarder_log'):
agentConfig['has_datadog'] = True
# Dogstream config
if config.has_option("Main", "dogstream_log"):
# Older version, single log support
log_path = config.get("Main", "dogstream_log")
if config.has_option("Main", "dogstream_line_parser"):
agentConfig["dogstreams"] = ':'.join(
[log_path,
config.get("Main", "dogstream_line_parser")])
else:
agentConfig["dogstreams"] = log_path
elif config.has_option("Main", "dogstreams"):
agentConfig["dogstreams"] = config.get("Main", "dogstreams")
if config.has_option("Main", "nagios_perf_cfg"):
agentConfig["nagios_perf_cfg"] = config.get(
"Main", "nagios_perf_cfg")
if config.has_section('WMI'):
agentConfig['WMI'] = {}
for key, value in config.items('WMI'):
agentConfig['WMI'][key] = value
if config.has_option("Main", "limit_memory_consumption") and \
config.get("Main", "limit_memory_consumption") is not None:
agentConfig["limit_memory_consumption"] = int(
config.get("Main", "limit_memory_consumption"))
else:
agentConfig["limit_memory_consumption"] = None
if config.has_option("Main", "skip_ssl_validation"):
agentConfig["skip_ssl_validation"] = _is_affirmative(
config.get("Main", "skip_ssl_validation"))
except ConfigParser.NoSectionError, e:
sys.stderr.write('Config file not found or incorrectly formatted.\n')
sys.exit(2)
def get_logging_config(cfg_path=None):
system_os = get_os()
logging_config = {
'log_level': None,
'log_to_event_viewer': False,
'log_to_syslog': False,
'syslog_host': None,
'syslog_port': None,
}
if system_os == 'windows':
logging_config['windows_collector_log_file'] = os.path.join(
_windows_commondata_path(), 'Datadog', 'logs', 'collector.log')
logging_config['windows_forwarder_log_file'] = os.path.join(
_windows_commondata_path(), 'Datadog', 'logs', 'forwarder.log')
logging_config['windows_dogstatsd_log_file'] = os.path.join(
_windows_commondata_path(), 'Datadog', 'logs', 'dogstatsd.log')
logging_config['jmxfetch_log_file'] = os.path.join(
_windows_commondata_path(), 'Datadog', 'logs', 'jmxfetch.log')
else:
logging_config['collector_log_file'] = '/var/log/datadog/collector.log'
logging_config['forwarder_log_file'] = '/var/log/datadog/forwarder.log'
logging_config['dogstatsd_log_file'] = '/var/log/datadog/dogstatsd.log'
logging_config['jmxfetch_log_file'] = '/var/log/datadog/jmxfetch.log'
logging_config['log_to_syslog'] = True
config_path = get_config_path(cfg_path, os_name=system_os)
config = ConfigParser.ConfigParser()
config.readfp(skip_leading_wsp(open(config_path)))
if config.has_section('handlers') or config.has_section(
'loggers') or config.has_section('formatters'):
if system_os == 'windows':
config_example_file = "https://github.com/DataDog/dd-agent/blob/master/packaging/datadog-agent/win32/install_files/datadog_win32.conf"
else:
config_example_file = "https://github.com/DataDog/dd-agent/blob/master/datadog.conf.example"
sys.stderr.write(
"""Python logging config is no longer supported and will be ignored.
To configure logging, update the logging portion of 'datadog.conf' to match:
'%s'.
""" % config_example_file)
for option in logging_config:
if config.has_option('Main', option):
logging_config[option] = config.get('Main', option)
levels = {
'CRITICAL': logging.CRITICAL,
'DEBUG': logging.DEBUG,
'ERROR': logging.ERROR,
'FATAL': logging.FATAL,
'INFO': logging.INFO,
'WARN': logging.WARN,
'WARNING': logging.WARNING,
}
if config.has_option('Main', 'log_level'):
logging_config['log_level'] = levels.get(
config.get('Main', 'log_level'))
if config.has_option('Main', 'log_to_syslog'):
logging_config['log_to_syslog'] = config.get(
'Main', 'log_to_syslog').strip().lower() in ['yes', 'true', 1]
if config.has_option('Main', 'log_to_event_viewer'):
logging_config['log_to_event_viewer'] = config.get(
'Main',
'log_to_event_viewer').strip().lower() in ['yes', 'true', 1]
if config.has_option('Main', 'syslog_host'):
host = config.get('Main', 'syslog_host').strip()
if host:
logging_config['syslog_host'] = host
else:
logging_config['syslog_host'] = None
if config.has_option('Main', 'syslog_port'):
port = config.get('Main', 'syslog_port').strip()
try:
logging_config['syslog_port'] = int(port)
except Exception:
logging_config['syslog_port'] = None
if config.has_option('Main', 'disable_file_logging'):
logging_config['disable_file_logging'] = config.get(
'Main',
'disable_file_logging').strip().lower() in ['yes', 'true', 1]
else:
logging_config['disable_file_logging'] = False
return logging_config
