def _add_conf_tar(self):
conf_path = get_config_path()
if self._can_read(conf_path, output=False):
self._add_clean_conf(
conf_path,
'etc',
self.MAIN_CREDENTIALS
)
if not Platform.is_windows():
supervisor_path = os.path.join(
os.path.dirname(get_config_path()),
'supervisor.conf'
)
if self._can_read(supervisor_path, output=False):
self._add_clean_conf(
supervisor_path,
'etc'
)
for file_path in glob.glob(os.path.join(get_confd_path(), '*.yaml')) +\
glob.glob(os.path.join(get_confd_path(), '*.yaml.default')):
if self._can_read(file_path, output=False):
self._add_clean_conf(
file_path,
os.path.join('etc', 'confd'),
self.CHECK_CREDENTIALS
)
def __getNewFileChoice():
dialog = gtk.FileChooserDialog("New file...", None,
gtk.FILE_CHOOSER_ACTION_SAVE, (gtk.STOCK_CANCEL,
gtk.RESPONSE_CANCEL, gtk.STOCK_OPEN, gtk.RESPONSE_OK))
filter = gtk.FileFilter()
filter.set_name("All files")
filter.add_pattern("*")
dialog.add_filter(filter)
# Does the user have a folder selected that we should instead load on
# newFile?
from leftpanel import leftview
model, iter = leftview.get_selection().get_selected()
from datastore import F_REF
try:
object = model.get_value(iter, F_REF)
if isinstance(object, PackageFileObj): # A file
folder = object.parent
elif isinstance(object, FolderObj): # A folder
folder = object
if folder == None:
folderPath = get_config_path()
else:
folderPath = folder.path # Get the path to the folder object
except TypeError,e:
print >>stderr, "__getNewFileChoice:",e
#Nothing selected, select default
folderPath = get_config_path()
def _add_conf_tar(self):
conf_path = get_config_path()
if self._can_read(conf_path):
self._add_file_tar(
self._strip_comment(conf_path),
os.path.join('etc', 'datadog.conf'),
original_file_path=conf_path
)
if not Platform.is_windows():
supervisor_path = os.path.join(
os.path.dirname(get_config_path()),
'supervisor.conf'
)
if self._can_read(supervisor_path):
self._add_file_tar(
self._strip_comment(supervisor_path),
os.path.join('etc', 'supervisor.conf'),
original_file_path=supervisor_path
)
for file_path in glob.glob(os.path.join(get_confd_path(), '*.yaml')) +\
glob.glob(os.path.join(get_confd_path(), '*.yaml.default')):
if self._can_read(file_path, output=False):
self._add_clean_confd(file_path)
def _add_conf_tar(self):
conf_path = get_config_path()
if self._can_read(conf_path):
self._add_file_tar(
self._strip_comment(conf_path),
os.path.join('etc', 'datadog.conf'),
original_file_path=conf_path
)
if not Platform.is_windows():
supervisor_path = os.path.join(
os.path.dirname(get_config_path()),
'supervisor.conf'
)
if self._can_read(supervisor_path):
self._add_file_tar(
self._strip_comment(supervisor_path),
os.path.join('etc', 'supervisor.conf'),
original_file_path=supervisor_path
)
for file_path in glob.glob(os.path.join(get_confd_path(), '*.yaml')) +\
glob.glob(os.path.join(get_confd_path(), '*.yaml.default')):
if self._can_read(file_path, output=False):
self._add_clean_confd(file_path)
def __getNewFileChoice():
dialog = gtk.FileChooserDialog("New file...", None,
gtk.FILE_CHOOSER_ACTION_SAVE,
(gtk.STOCK_CANCEL, gtk.RESPONSE_CANCEL,
gtk.STOCK_OPEN, gtk.RESPONSE_OK))
filter = gtk.FileFilter()
filter.set_name("All files")
filter.add_pattern("*")
dialog.add_filter(filter)
# Does the user have a folder selected that we should instead load on
# newFile?
from leftpanel import leftview
model, iter = leftview.get_selection().get_selected()
from datastore import F_REF
try:
object = model.get_value(iter, F_REF)
if isinstance(object, PackageFileObj): # A file
folder = object.parent
elif isinstance(object, FolderObj): # A folder
folder = object
if folder == None:
folderPath = get_config_path()
else:
folderPath = folder.path # Get the path to the folder object
except TypeError, e:
print >> stderr, "__getNewFileChoice:", e
#Nothing selected, select default
folderPath = get_config_path()
def _add_conf_tar(self):
conf_path = get_config_path()
if self._can_read(conf_path, output=False):
self._add_clean_conf(
conf_path,
'etc',
self.MAIN_CREDENTIALS
)
if not Platform.is_windows():
supervisor_path = os.path.join(
os.path.dirname(get_config_path()),
'supervisor.conf'
)
if self._can_read(supervisor_path, output=False):
self._add_clean_conf(
supervisor_path,
'etc'
)
for file_path in glob.glob(os.path.join(get_confd_path(), '*.yaml')) +\
glob.glob(os.path.join(get_confd_path(), '*.yaml.default')):
if self._can_read(file_path, output=False):
self._add_clean_conf(
file_path,
os.path.join('etc', 'confd'),
self.CHECK_CREDENTIALS
)
def main():
logging.basicConfig(level=logging.INFO)
if isfile(get_config_path(__file__, '/esxi.ini')):
config = ConfigurationINI(get_config_path(__file__, '/esxi.ini'))
elif isfile('/etc/esxi.ini'):
config = ConfigurationINI('/etc/esxi.ini')
else:
logging.critical('/etc/esxi.ini missing.')
exit(0)
logging.debug('Configuration file used : {conf}'.format(conf=get_config_path(__file__, '/esxi.ini')))
try:
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.connect(config['esxi']['hostip'], username=config['esxi']['username'], password=config['esxi']['password'])
except SocketError as e:
logging.critical('Host unreachable.')
logging.critical(e.__str__())
exit()
logging.info('vim-cmd hostsvc/firmware/sync_config')
ssh.exec_command('vim-cmd hostsvc/firmware/sync_config')
logging.info('vim-cmd hostsvc/firmware/backup_config')
stdin, stdout, stderr = ssh.exec_command('vim-cmd hostsvc/firmware/backup_config')
for l in stdout:
m = search('http://\*(.*)', l.strip())
if m is not None:
download = "http://{host}{position}".format(
host=config['esxi']['hostip'],
position=m.group(1)
)
logging.info("Downloading {0}".format(download))
local_file = '{localpath}/backup-{host}-{date}.tgz'.format(
host=config['esxi']['hostdns'],
date=strftime(config['local']['dateformat']),
localpath=config['local']['savepath']
)
urlretrieve(download, local_file)
if config['webdav']['enabled']:
logging.info("Uploading file on WebDAV")
comediaoc = webdav_connect(
config['webdav']['host'],
username=config['webdav']['username'],
password=config['webdav']['password'],
protocol=config['webdav']['proto'],
verify_ssl=False
)
comediaoc.upload(local_file, '{0}/backup-{1}-{2}.tgz'.format(
config['webdav']['savepath'],
config['esxi']['hostdns'],
strftime(config['local']['dateformat'])
))
ssh.close()
def main():
logging.basicConfig(level=logging.INFO)
if isfile(get_config_path(__file__, '/esxi.ini')):
config = ConfigurationINI(get_config_path(__file__, '/esxi.ini'))
elif isfile('/etc/esxi.ini'):
config = ConfigurationINI('/etc/esxi.ini')
else:
logging.critical('/etc/esxi.ini missing.')
exit(0)
logging.debug('Configuration file used : {conf}'.format(
conf=get_config_path(__file__, '/esxi.ini')))
try:
ssh = SSHClient()
ssh.set_missing_host_key_policy(AutoAddPolicy())
ssh.connect(config['esxi']['hostip'],
username=config['esxi']['username'],
password=config['esxi']['password'])
except SocketError as e:
logging.critical('Host unreachable.')
logging.critical(e.__str__())
exit()
logging.info('vim-cmd hostsvc/firmware/sync_config')
ssh.exec_command('vim-cmd hostsvc/firmware/sync_config')
logging.info('vim-cmd hostsvc/firmware/backup_config')
stdin, stdout, stderr = ssh.exec_command(
'vim-cmd hostsvc/firmware/backup_config')
for l in stdout:
m = search('http://\*(.*)', l.strip())
if m is not None:
download = "http://{host}{position}".format(
host=config['esxi']['hostip'], position=m.group(1))
logging.info("Downloading {0}".format(download))
local_file = '{localpath}/backup-{host}-{date}.tgz'.format(
host=config['esxi']['hostdns'],
date=strftime(config['local']['dateformat']),
localpath=config['local']['savepath'])
urlretrieve(download, local_file)
if config['webdav']['enabled']:
logging.info("Uploading file on WebDAV")
comediaoc = webdav_connect(
config['webdav']['host'],
username=config['webdav']['username'],
password=config['webdav']['password'],
protocol=config['webdav']['proto'],
verify_ssl=False)
comediaoc.upload(
local_file, '{0}/backup-{1}-{2}.tgz'.format(
config['webdav']['savepath'],
config['esxi']['hostdns'],
strftime(config['local']['dateformat'])))
ssh.close()
def _add_conf_tar(self):
conf_path = get_config_path()
log.info(" * {0}".format(conf_path))
self._tar.add(self._strip_comment(conf_path), os.path.join(self._prefix, "etc", "datadog.conf"))
if not Platform.is_windows():
supervisor_path = os.path.join(os.path.dirname(get_config_path()), "supervisor.conf")
log.info(" * {0}".format(supervisor_path))
self._tar.add(self._strip_comment(supervisor_path), os.path.join(self._prefix, "etc", "supervisor.conf"))
for file_path in glob.glob(os.path.join(get_confd_path(), "*.yaml")) + glob.glob(
os.path.join(get_confd_path(), "*.yaml.default")
):
self._add_clean_confd(file_path)
def _add_conf_tar(self):
conf_path = get_config_path()
log.info(" * {0}".format(conf_path))
self._tar.add(self._strip_comment(conf_path),
os.path.join(self._prefix, 'etc', 'datadog.conf'))
if not Platform.is_windows():
supervisor_path = os.path.join(os.path.dirname(get_config_path()),
'supervisor.conf')
log.info(" * {0}".format(supervisor_path))
self._tar.add(self._strip_comment(supervisor_path),
os.path.join(self._prefix, 'etc', 'supervisor.conf'))
for file_path in glob.glob(os.path.join(get_confd_path(), '*.yaml')):
self._add_clean_confd(file_path)
def _get_raw_config(self):
raw_config = {}
try:
config = ConfigParser.ConfigParser()
config_path = os.path.dirname(get_config_path())
if os.path.exists(os.path.join(config_path, "plugins.d")):
config_path = os.path.join(config_path, "plugins.d")
else:
config_path = os.path.join(config_path, "plugins.cfg")
if not os.access(config_path, os.R_OK):
self.logger.error("Unable to read the config file at " + config_path)
self.logger.error("Using no config...")
return raw_config
if os.path.isdir(config_path):
for config_file in glob(os.path.join(config_path, "*.cfg")):
config.read(config_file)
else:
config.read(config_path)
for section in config.sections():
raw_config[section] = {}
for option in config.options(section):
raw_config[section][option] = config.get(section, option)
except ConfigParser.ParsingError:
self.logger.error("v1 Plugins config file not found or incorrectly formatted.")
return raw_config
def get_smtp_config():
try:
config_file = config.get_config_path(config.EMAIL_CONF_PATH)
with open(config_file, 'r') as stream:
return yaml.load(stream)
except IOError as e:
LOG.error('[ERROR] file: {0}, reason: {1}\n'.format(config_file, repr(e)))
def _update_config_file(self, config):
log.debug('Querying registry to get missing config options')
registry_conf = get_registry_conf(config)
config.update(registry_conf)
if registry_conf:
log.info('Updating conf file options: %s', registry_conf.keys())
update_conf_file(registry_conf, get_config_path())
def __init__(self, instance):
"""The constructor."""
self.instance = instance
self.uuidmap_file = get_config_path('uuidmap/' + self.instance + '.json')
self.log = logging.getLogger('AgoConnection')
broker = str(get_config_option("system", "broker", "localhost"))
username = str(get_config_option("system", "username", "agocontrol"))
password = str(get_config_option("system", "password", "letmein"))
self.log.debug("Connecting to broker %s", broker)
self.connection = Connection(broker,
username=username, password=password, reconnect=True)
self.connection.open()
self.session = self.connection.session()
self.receiver = self.session.receiver(
"agocontrol; {create: always, node: {type: topic}}")
self.sender = self.session.sender(
"agocontrol; {create: always, node: {type: topic}}")
self.devices = {}
self.uuids = {}
self.handler = None
self.eventhandler = None
self.agocontroller = None
self.load_uuid_map()
def _update_config_file(self, config):
log.debug('Querying registry to get missing config options')
registry_conf = get_registry_conf(config)
config.update(registry_conf)
if registry_conf:
log.info('Updating conf file options: %s', registry_conf.keys())
update_conf_file(registry_conf, get_config_path())
def init6(agent_config=None, config_path=None, args=None):
if (not agent_config['use_dogstatsd'] and
(args and args[0] in ['start', 'restart'] or not args)):
log.info("Dogstatsd is disabled. Exiting")
# We're exiting purposefully, so exit with zero (supervisor's expected
# code). HACK: Sleep a little bit so supervisor thinks we've started cleanly
# and thus can exit cleanly.
sleep(4)
sys.exit(0)
env = copy.deepcopy(os.environ)
if agent_config.get('api_key'):
env['DD_API_KEY'] = str(agent_config['api_key'])
if agent_config.get('dogstatsd_port'):
env['DD_DOGSTATSD_PORT'] = str(agent_config['dogstatsd_port'])
if agent_config.get('dd_url'):
env['DD_DD_URL'] = str(agent_config['dd_url'])
if agent_config.get('non_local_traffic'):
env['DD_DOGSTATSD_NON_LOCAL_TRAFFIC'] = str(agent_config['non_local_traffic'])
if agent_config.get('dogstatsd_socket'):
env['DD_DOGSTATSD_SOCKET'] = str(agent_config['dogstatsd_socket'])
if agent_config.get('dogstatsd6_stats_port'):
env['DD_DOGSTATSD_STATS_PORT'] = str(agent_config['dogstatsd6_stats_port'])
env['DD_LOG_LEVEL'] = agent_config.get('log_level', 'info')
env['DD_CONF_PATH'] = os.path.join(
os.path.dirname(get_config_path(cfg_path=config_path)), "stackstate.yaml")
# metadata is sent by the collector, disable it in dogstatsd6 to avoid sending conflicting metadata
env['DD_ENABLE_METADATA_COLLECTION'] = 'false'
legacy_dogstatsd_log = get_logging_config().get('dogstatsd_log_file')
if legacy_dogstatsd_log:
env['DD_LOG_FILE'] = os.path.join(
os.path.dirname(legacy_dogstatsd_log), '{}.log'.format(Dogstatsd6.DSD6_BIN_NAME))
return Dogstatsd6._get_dsd6_path(), env
def init6(agent_config=None, config_path=None, args=None):
if (not agent_config['use_sdstatsd'] and
(args and args[0] in ['start', 'restart'] or not args)):
log.info("Sdstatsd is disabled. Exiting")
# We're exiting purposefully, so exit with zero (supervisor's expected
# code). HACK: Sleep a little bit so supervisor thinks we've started cleanly
# and thus can exit cleanly.
sleep(4)
sys.exit(0)
env = copy.deepcopy(os.environ)
if agent_config.get('agent_key'):
env['SD_AGENT_KEY'] = str(agent_config['agent_key'])
if agent_config.get('sdstatsd_port'):
env['SD_SDSTATSD_PORT'] = str(agent_config['sdstatsd_port'])
if agent_config.get('sd_url'):
env['SD_SD_URL'] = str(agent_config['sd_url'])
if agent_config.get('non_local_traffic'):
env['SD_SDSTATSD_NON_LOCAL_TRAFFIC'] = str(agent_config['non_local_traffic'])
if agent_config.get('sdstatsd_socket'):
env['SD_SDSTATSD_SOCKET'] = str(agent_config['sdstatsd_socket'])
if agent_config.get('sdstatsd6_stats_port'):
env['SD_SDSTATSD_STATS_PORT'] = str(agent_config['sdstatsd6_stats_port'])
env['SD_LOG_LEVEL'] = agent_config.get('log_level', 'info')
env['SD_CONF_PATH'] = os.path.join(
os.path.dirname(get_config_path(cfg_path=config_path)), "config.cfg")
# metadata is sent by the collector, disable it in sdstatsd6 to avoid sending conflicting metadata
env['SD_ENABLE_METADATA_COLLECTION'] = 'false'
legacy_sdstatsd_log = get_logging_config().get('sdstatsd_log_file')
if legacy_sdstatsd_log:
env['SD_LOG_FILE'] = os.path.join(
os.path.dirname(legacy_sdstatsd_log), '{}.log'.format(Sdstatsd6.SDSD6_BIN_NAME))
return Sdstatsd6._get_sdsd6_path(), env
def _get_raw_config(self):
raw_config = {}
try:
config = ConfigParser.ConfigParser()
config_path = os.path.dirname(get_config_path())
if os.path.exists(os.path.join(config_path, 'plugins.d')):
config_path = os.path.join(config_path, 'plugins.d')
else:
config_path = os.path.join(config_path, 'plugins.cfg')
if not os.access(config_path, os.R_OK):
self.logger.error('Unable to read the config file at ' +
config_path)
self.logger.error('Using no config...')
return raw_config
if os.path.isdir(config_path):
for config_file in glob(os.path.join(config_path, "*.cfg")):
config.read(config_file)
else:
config.read(config_path)
for section in config.sections():
raw_config[section] = {}
for option in config.options(section):
raw_config[section][option] = config.get(section, option)
except ConfigParser.ParsingError:
self.logger.error(
"v1 Plugins config file not found or incorrectly formatted.")
return raw_config
def __new__(cls):
if cls._instance is None:
cls._instance = object.__new__(cls)
conf = config.load_config(config.get_config_path(), {'postgres.port': 5432})
postgres_config = {
'host': conf['postgres.host'],
'port': conf['postgres.port'],
'dbname': conf['postgres.database'],
'user': conf['postgres.user'],
'password': conf['postgres.password']
}
try:
logging.info('Connecting to postgres database...')
connection = PostgresDB._instance.connection = psycopg2.connect(**postgres_config)
cursor = PostgresDB._instance.cursor = connection.cursor()
cursor.execute('select version()')
db_version = cursor.fetchone()
except Exception as conn_err:
logging.exception('connection problem: %s', conn_err)
PostgresDB._instance = None
else:
logging.info('connection established\n%s', db_version[0])
return cls._instance
def __init__(self, directory='/home'):
self.init_docs_folder(directory)
if not os.path.exists(get_config_path()):
self.authenticate()
self.token = load_config()['token']
def __init__(self, parent=None):
QSplitter.__init__(self, parent)
self.setWindowTitle(MAIN_WINDOW_TITLE)
self.setWindowIcon(get_icon("agent.svg"))
self.sysTray = SystemTray(self)
self.connect(self.sysTray, SIGNAL(
"activated(QSystemTrayIcon::ActivationReason)"), self.__icon_activated)
checks = get_checks()
datadog_conf = DatadogConf(
get_config_path(), description="Agent settings file: datadog.conf")
self.log_file = LogFile()
listwidget = QListWidget(self)
listwidget.addItems(
[osp.basename(check.module_name).replace("_", " ").title() for check in checks])
self.properties = PropertiesWidget(self)
self.addWidget(listwidget)
self.addWidget(self.properties)
self.connect(self.properties.enable_button, SIGNAL("clicked()"),
lambda: enable_check(self.properties))
self.connect(self.properties.disable_button, SIGNAL("clicked()"),
lambda: disable_check(self.properties))
self.connect(self.properties.save_button, SIGNAL("clicked()"),
lambda: save_file(self.properties))
self.connect(listwidget, SIGNAL('currentRowChanged(int)'),
lambda row: self.properties.set_item(checks[row]))
self.connect(self.properties.edit_datadog_conf_button, SIGNAL('clicked()'),
lambda: self.properties.set_datadog_conf(datadog_conf))
self.connect(self.properties.view_log_button, SIGNAL('clicked()'),
lambda: self.properties.set_log_file(self.log_file))
self.manager_menu = Menu(self)
self.connect(self.properties.menu_button, SIGNAL("clicked()"),
lambda: self.manager_menu.popup(self.properties.menu_button.mapToGlobal(QPoint(0, 0))))
listwidget.setCurrentRow(0)
self.setSizes([150, 1])
self.setStretchFactor(1, 1)
self.resize(QSize(950, 600))
self.properties.set_datadog_conf(datadog_conf)
self.do_refresh()
def __init__(self, parent=None):
QSplitter.__init__(self, parent)
self.setWindowTitle(MAIN_WINDOW_TITLE)
self.setWindowIcon(get_icon("agent.svg"))
self.sysTray = SystemTray(self)
self.connect(self.sysTray, SIGNAL("activated(QSystemTrayIcon::ActivationReason)"), self.__icon_activated)
checks = get_checks()
datadog_conf = DatadogConf(get_config_path(), description="Agent settings file: datadog.conf")
self.log_file = LogFile()
listwidget = QListWidget(self)
listwidget.addItems([osp.basename(check.module_name).replace("_", " ").title() for check in checks])
self.properties = PropertiesWidget(self)
self.addWidget(listwidget)
self.addWidget(self.properties)
self.connect(self.properties.enable_button, SIGNAL("clicked()"),
lambda: enable_check(self.properties))
self.connect(self.properties.disable_button, SIGNAL("clicked()"),
lambda: disable_check(self.properties))
self.connect(self.properties.save_button, SIGNAL("clicked()"),
lambda: save_file(self.properties))
self.connect(listwidget, SIGNAL('currentRowChanged(int)'),
lambda row: self.properties.set_item(checks[row]))
self.connect(self.properties.edit_datadog_conf_button, SIGNAL('clicked()'),
lambda: self.properties.set_datadog_conf(datadog_conf))
self.connect(self.properties.view_log_button, SIGNAL('clicked()'),
lambda: self.properties.set_log_file(self.log_file))
self.manager_menu = Menu(self)
self.connect(self.properties.menu_button, SIGNAL("clicked()"),
lambda: self.manager_menu.popup(self.properties.menu_button.mapToGlobal(QPoint(0,0))))
listwidget.setCurrentRow(0)
self.setSizes([150, 1])
self.setStretchFactor(1, 1)
self.resize(QSize(950, 600))
self.properties.set_datadog_conf(datadog_conf)
self.do_refresh()
def _update_config_file(self, config):
log.debug('Querying registry to get missing config options')
registry_conf = get_registry_conf(config)
config.update(registry_conf)
if registry_conf:
log.info('Updating conf file options: %s', registry_conf.keys())
try:
update_conf_file(registry_conf, get_config_path())
log.info('update succeeded, deleting old values')
remove_registry_conf()
except Exception:
log.warning('Failed to update config file; registry configuration persisted')
def _update_config_file(self, config):
log.debug('Querying registry to get missing config options')
registry_conf = get_registry_conf(config)
config.update(registry_conf)
if registry_conf:
log.info('Updating conf file options: %s', registry_conf.keys())
try:
update_conf_file(registry_conf, get_config_path())
log.info('update succeeded, deleting old values')
remove_registry_conf()
except Exception:
log.warning('Failed to update config file; registry configuration persisted')
def initData():
""" Constructs Folder and PackageFile objects """
path = get_config_path()
#print "USING PATH: " + path
for rootDir, folders, files in os.walk(path, topdown=True):
# Begin the construction
if rootDir is get_config_path(): # we are top level, no children
# Filter unrelated files TOPLEVEL ONLY
# Folders sanity
for folder in folders[:]:
if folder not in config_files:
folders.remove(folder) #ignore unrelated folders
# Files sanity
for pfile in files[:]:
if pfile not in config_files:
files.remove(pfile) #ignore unrelated files
#construct
for folder in folders:
foldobj = FolderObj.FolderObj(folder, rootDir + "/" + folder)
TLFolders.append(foldobj)
for pfile in files:
fileobj = PackageFileObj.PackageFileObj(
pfile, rootDir + "/" + pfile, None)
TLFiles.append(fileobj)
else: # No longer top level, we are inside a folder
tlname = rootDir.split("/")[-1] # /etc/portage/sets => sets
#construct
for folder in folders: # recursive folder
parent = __getParent(tlname, "folder") # FolderObj
foldobj = FolderObj.FolderObj(folder, rootDir + "/" + folder)
foldobj.parentState = True
foldobj.parent = parent
parent.addFolder(foldobj)
parent.childState = True
TLFolders.append(foldobj)
for pfile in files:
parent = __getParent(tlname, "file") #PackageFileObj
fileobj = PackageFileObj.PackageFileObj(
pfile, rootDir + "/" + pfile, parent)
parent.addPackage(fileobj)
def initData():
""" Constructs Folder and PackageFile objects """
path = get_config_path()
#print "USING PATH: " + path
for rootDir, folders, files in os.walk(path, topdown=True):
# Begin the construction
if rootDir is get_config_path(): # we are top level, no children
# Filter unrelated files TOPLEVEL ONLY
# Folders sanity
for folder in folders[:]:
if folder not in config_files:
folders.remove(folder) #ignore unrelated folders
# Files sanity
for pfile in files[:]:
if pfile not in config_files:
files.remove(pfile) #ignore unrelated files
#construct
for folder in folders:
foldobj = FolderObj.FolderObj(folder, rootDir + "/" + folder)
TLFolders.append(foldobj)
for pfile in files:
fileobj = PackageFileObj.PackageFileObj(pfile, rootDir + "/" + pfile, None)
TLFiles.append(fileobj)
else: # No longer top level, we are inside a folder
tlname = rootDir.split("/")[-1] # /etc/portage/sets => sets
#construct
for folder in folders: # recursive folder
parent = __getParent(tlname, "folder") # FolderObj
foldobj = FolderObj.FolderObj(folder, rootDir + "/" + folder)
foldobj.parentState = True
foldobj.parent = parent
parent.addFolder(foldobj)
parent.childState = True
TLFolders.append(foldobj)
for pfile in files:
parent = __getParent(tlname, "file") #PackageFileObj
fileobj = PackageFileObj.PackageFileObj(pfile, rootDir + "/" + pfile, parent)
parent.addPackage(fileobj)
class CacheList:
conf_ttl = int(
config.load_config(config.get_config_path(),
{'cache.expired': 10})['cache.expired'])
conf_ttl *= 60 # in minutes
def __init__(self):
self.entries = []
self.lock = RLock()
def add_entry(self, key, value, ttl=conf_ttl):
with self.lock:
self.entries.append(CacheEntry(key, value, ttl))
def read_entries(self):
with self.lock:
self.entries = list(
itertools.dropwhile(lambda x: x.expired(), self.entries))
return self.entries
def renameFile(*args):
""" Renames the currently selected file """
from leftpanel import leftview
model, iter = leftview.get_selection().get_selected()
from datastore import F_REF
try:
object = model.get_value(iter, F_REF)
if isinstance(object, PackageFileObj): # A file
type = "File"
if object.parent == None:
setFolder = get_config_path()
else:
setFolder = object.parent.path
elif isinstance(object, FolderObj): # A folder
type = "Directory"
setFolder = object.parent.path
if ensureNotModified(msg):
__createRenameDialog(object, type, setFolder)
except TypeError, e:
print >> stderr, "__renameFile:", e
def statistic():
stat_count_path = 'statistic.count'
conf = config.load_config(config.get_config_path(), {stat_count_path: 5})
query_stmt = """
SELECT r.region_id, region_name, COUNT(region_name)
FROM regions r JOIN cities c ON r.region_id = c.region_id
JOIN comments cm ON c.city_id = cm.city_id
GROUP BY r.region_id
HAVING COUNT(r.region_id) > {COUNT}
ORDER BY COUNT(region_name) DESC;
"""
try:
header_only = False
comment_statistics = []
db_instance = db.PostgresDB()
for region_stat in db_instance.query(
query_stmt.format(COUNT=conf[stat_count_path])):
region_statistic = collections.OrderedDict()
url = Markup('<a href="{}">{}</a>'.format(
url_for('get_comments', regions=region_stat[0]),
region_stat[1]))
region_statistic['Region'] = url
region_statistic['Comments'] = region_stat[2]
comment_statistics.append(region_statistic)
if not len(comment_statistics):
comment_statistics.append(
collections.OrderedDict(Region='', Comments=''))
header_only = True
print(comment_statistics)
return render_template('stat.html',
comment_statistics=comment_statistics,
header_only=header_only)
except:
flash('Internal service problem:(', category='alert-danger')
return redirect(url_for('comments'))
def renameFile(*args):
""" Renames the currently selected file """
from leftpanel import leftview
model, iter = leftview.get_selection().get_selected()
from datastore import F_REF
try:
object = model.get_value(iter, F_REF)
if isinstance(object, PackageFileObj): # A file
type = "File"
if object.parent == None:
setFolder = get_config_path()
else:
setFolder = object.parent.path
elif isinstance(object, FolderObj): # A folder
type = "Directory"
setFolder = object.parent.path
if ensureNotModified(msg):
__createRenameDialog(object, type, setFolder)
except TypeError, e:
print >> stderr, "__renameFile:", e
def __init__(self, parent=None):
QSplitter.__init__(self, parent)
self.setWindowTitle(MAIN_WINDOW_TITLE)
self.setWindowIcon(get_icon("agent.svg"))
self.sysTray = SystemTray(self)
self.connect(self.sysTray, SIGNAL("activated(QSystemTrayIcon::ActivationReason)"), self.__icon_activated)
checks = get_checks()
datadog_conf = DatadogConf(get_config_path(), description="Agent settings file: datadog.conf")
self.forwarder_log_file = ForwarderLogFile()
self.collector_log_file = CollectorLogFile()
self.dogstatsd_log_file = DogstatsdLogFile()
self.jmxfetch_log_file = JMXFetchLogFile()
listwidget = QListWidget(self)
listwidget.addItems([osp.basename(check.module_name).replace("_", " ").title() for check in checks])
self.properties = PropertiesWidget(self)
self.setting_button = QPushButton(get_icon("info.png"), "Logs and Status", self)
self.menu_button = QPushButton(get_icon("settings.png"), "Actions", self)
self.settings = [
(
"Forwarder Logs",
lambda: [
self.properties.set_log_file(self.forwarder_log_file),
self.show_html(self.properties.group_code, self.properties.html_window, False),
],
),
(
"Collector Logs",
lambda: [
self.properties.set_log_file(self.collector_log_file),
self.show_html(self.properties.group_code, self.properties.html_window, False),
],
),
(
"Dogstatsd Logs",
lambda: [
self.properties.set_log_file(self.dogstatsd_log_file),
self.show_html(self.properties.group_code, self.properties.html_window, False),
],
),
(
"JMX Fetch Logs",
lambda: [
self.properties.set_log_file(self.jmxfetch_log_file),
self.show_html(self.properties.group_code, self.properties.html_window, False),
],
),
(
"Agent Status",
lambda: [
self.properties.html_window.setHtml(self.properties.html_window.latest_status()),
self.show_html(self.properties.group_code, self.properties.html_window, True),
self.properties.set_status(),
],
),
]
self.agent_settings = QPushButton(get_icon("edit.png"), "Settings", self)
self.connect(
self.agent_settings,
SIGNAL("clicked()"),
lambda: [
self.properties.set_datadog_conf(datadog_conf),
self.show_html(self.properties.group_code, self.properties.html_window, False),
],
)
self.setting_menu = SettingMenu(self.settings)
self.connect(
self.setting_button,
SIGNAL("clicked()"),
lambda: self.setting_menu.popup(self.setting_button.mapToGlobal(QPoint(0, 0))),
)
self.manager_menu = Menu(self)
self.connect(
self.menu_button,
SIGNAL("clicked()"),
lambda: self.manager_menu.popup(self.menu_button.mapToGlobal(QPoint(0, 0))),
)
holdingBox = QGroupBox("", self)
Box = QVBoxLayout(self)
Box.addWidget(self.agent_settings)
Box.addWidget(self.setting_button)
Box.addWidget(self.menu_button)
Box.addWidget(listwidget)
holdingBox.setLayout(Box)
self.addWidget(holdingBox)
self.addWidget(self.properties)
self.connect(self.properties.enable_button, SIGNAL("clicked()"), lambda: enable_check(self.properties))
self.connect(self.properties.disable_button, SIGNAL("clicked()"), lambda: disable_check(self.properties))
self.connect(self.properties.save_button, SIGNAL("clicked()"), lambda: save_file(self.properties))
self.connect(
self.properties.refresh_button,
SIGNAL("clicked()"),
lambda: [
self.properties.set_log_file(self.properties.current_file),
self.properties.html_window.setHtml(self.properties.html_window.latest_status()),
],
)
self.connect(
listwidget,
SIGNAL("currentRowChanged(int)"),
lambda row: [
self.properties.set_item(checks[row]),
self.show_html(self.properties.group_code, self.properties.html_window, False),
],
)
listwidget.setCurrentRow(0)
self.setSizes([150, 1])
self.setStretchFactor(1, 1)
self.resize(QSize(950, 600))
self.properties.set_datadog_conf(datadog_conf)
self.do_refresh()
import logging
import os
import sys
import config
from app import routes
default_log_level = logging.DEBUG
default_log_file = 'app.log'
default_log_entry_format = '%(filename)s[LINE:%(lineno)d]# %(levelname)-8s [%(asctime)s] %(message)s'
if __name__ == "__main__":
config_file_path = config.get_config_path()
if not os.path.exists(config_file_path):
logging.fatal('Configuration file %s not found', config_file_path)
sys.exit(1)
conf = config.load_config(config_file_path)
log_file_handler = logging.FileHandler(
conf.get('logger.file', default_log_file), 'a')
formatter = logging.Formatter(
conf.get('logger.format', default_log_entry_format))
log_file_handler.setFormatter(formatter)
log_file_handler.setLevel(conf.get('logger.level', default_log_level))
stream_log_handler = logging.StreamHandler()
stream_log_handler.setFormatter(
logging.Formatter(default_log_entry_format))
handler.setFormatter(formatter)
logger.addHandler(handler)
return logger
def clear_out_logger(logger):
handlers = logger.handlers[:]
for handler in handlers:
handler.close()
logger.removeHandler(handler)
if __name__ == '__main__':
print('\n<<< Start >>>\r\n')
# read in config file
config_file = config.get_config_path()
Config = configparser.ConfigParser()
Config.read(config_file)
# << PARAMETERS >>
str_reachid = Config['reach and order']['reach_id']
str_orderid = Config['reach and order']['order_id']
# Cross section method:
parm_ivert = float(Config['cross section method']['parm_ivert']) # 0.2 default
XnPtDist = int(
Config['cross section method']['XnPtDist']) # 3 is default Step along Xn length for interpolating elevation
parm_ratiothresh = float(Config['cross section method']['parm_ratiothresh']) # 1.5 default
parm_slpthresh = float(Config['cross section method']['parm_slpthresh']) # 0.03 default
p_fpxnlen = int(Config['cross section method']['p_fpxnlen']) # 2D cross-section method (assign by order?)
# p_buffxnlen = Config['cross section method']['p_buffxnlen'] # meters Hardcoded in: get_xn_length_by_order()
import os
import json
import shutil
from tqdm import tqdm
import irsg_core.data_pull as dp
from config import get_config_path
with open(get_config_path()) as f:
cfg_data = json.load(f)
sg_path = cfg_data['file_paths']['sg_path']
def copy_images_indexed(path, if_data):
from_path = os.path.join(path, 'sg_test_images')
to_path = os.path.join(path, 'by_index')
if not os.path.exists(to_path):
os.makedirs(to_path)
for index, image_data in tqdm(enumerate(if_data.vg_data),
total=len(if_data.vg_data)):
image_name = os.path.basename(image_data.image_path)
image_path_from = os.path.join(from_path, image_name)
image_path_to = os.path.join(to_path, 'i{:03d}.jpg'.format(index))
shutil.copy(image_path_from, image_path_to)
if __name__ == '__main__':
if_data = dp.get_ifdata(use_csv=True)
copy_images_indexed(sg_path, if_data)
time.sleep(10)
upload_local_changes()
if __name__ == "__main__":
argparser = argparse.ArgumentParser()
argparser.add_argument("-d", "--debug", help="turn on debug mode", action="store_true")
argparser.add_argument("-c", "--config", help="specify config file path")
args = argparser.parse_args()
if args.debug:
debug_mode = True
config_path = ""
if not args.config:
config_path = config.get_config_path()
else:
config_path = args.config
try:
client_config = load_config(config_path)
except IOError:
print "Unable to load config file. Exiting."
sys.exit(-1)
if "server" in client_config:
server_url = client_config["server"]
if server_url[-1] == '/':
server_url = server_url[0:-1]
if "folder" in client_config:
from pathlib import Path
from timeit import default_timer as timer
import sys
import pandas as pd
import config
import funcs
from post_processing import post_process
if __name__ == '__main__':
print('\n<<< Start >>>\r\n')
# read in config file & generate PARAMS dict
CONFIG_FILE = config.get_config_path()
PARAMS = config.validate_config_params(CONFIG_FILE)
# generate list of huc folders in data dir
HUC_LST = []
for p in PARAMS['data_dir'].iterdir():
if p.is_dir():
HUC_LST.append(p)
# loop through the HUC folders
for huc_dir in HUC_LST:
HUCID = str(huc_dir).split('\\')[-1]
# start HUC processing time
start_time_i = timer()
def main():
intro_msg = '''
+-+-+-+-+-+ +-+-+-+-+-+
|F|A|C|E|T| |S|E|T|U|P|
+-+-+-+-+-+ +-+-+-+-+-+
setup.py script will automatically download ancillary datasets needed by FACET from Science Base.
It also updates the config file with ancillary data file paths as well.
The script takes anywhere from 30 minutes to 1-2 hours to run successfully depending on your internet speed.\n
'''
start = timer()
# arg parser
parser = argparse.ArgumentParser(prog='FACET set-up script',
description=intro_msg)
# user arg defined project path
parser.add_argument('--path',
'-p',
help='full path for project',
required=True)
args = parser.parse_args()
# intro message
print(intro_msg)
# project pathname
project_dir = Path(args.path)
# config file path
config_ini_file = config.get_config_path()
# get science base json data
sb_json = json_to_dict(project_dir)
### STEP 1
# download, extract and clean up
st_step_01 = timer()
print('''
Step #1: Downloading and extracting ancillary datasets. This will take 30-45 mins. Please wait.
''')
# number of items to download at any given time
no_of_items = int(len(sb_json) / 3)
pool = mp.Pool(processes=no_of_items)
pool.map(get_data, [(v) for v in sb_json.values()])
pool.close()
end_step_01 = round((timer() - st_step_01) / 60.0, 2)
print(f'Step #1 complete. Time elapsed: {end_step_01} mins\n')
### STEP 2
st_step_02 = timer()
# update and save config file
print('Step #2: Updating config file and deleting temp files.')
ini_dict = update_config(config_ini_file, sb_json, project_dir)
# clean up zip files
for v in sb_json.values():
if v['file'].is_file():
os.remove(v['file'])
# result output for users
end_step_02 = round((timer() - st_step_02) / 60.0, 2)
print(f'Step #2 complete. Time elapsed: {end_step_02} mins\n')
files_downloaded = list(project_dir.rglob('*/ancillary_data/*.shp'))
end = round((timer() - start) / 60.0, 2)
# message 1
msg_1 = '''
Setup complete. See summary below:
1. Files downloaded:
'''
# message 2
msg_2 = f'''
2. Following variables were updated in config file:
taudem: {ini_dict['pre process dem']['taudem']}
taudem: {ini_dict['pre process dem']['taudem cores']}
physio cbw: {ini_dict['file paths']['physio cbw']}
physio drb: {ini_dict['file paths']['physio drb']}
census roads: {ini_dict['file paths']['census roads']}
census rails: {ini_dict['file paths']['census rails']}
*************************************************
*** CAUTION: READ THE SECTION BELOW CAREFULLY ***
*************************************************
Users need to open config.ini, review and edit the following parameters.
For more details please refer to README, it will provide a detailed overview of each
parameter and if it can be modified or not:
* data_dir: This is a sub-directory inside your project directory where your data is stored
* spatial ref: needs to be in PROJ.4 string (see README)
* Under 'pre process dem':
- taudem cores (default is set to using all cores, but users can manually change it)
- resample resolution (see README)
* breach options: 'pre-condition dem & fast-breach' is the default option (see README)
* post processing: 'r exe path' needs to be changed based on your local installation of R
DO NOT update values under the following subsections in config.ini (see README):
* reach and order
* cross section method
* width from curvature via buff. method
Total time elapsed: {end} mins'''
for i in [msg_1, files_downloaded, msg_2]:
if isinstance(i, str):
print(i)
if isinstance(i, list):
for x in i:
print(f'\t{x}')
def cli(add, rm, show, complete, dotfiles, configs, packages, fonts, old_path, new_path, remote, reinstall_packages,
reinstall_configs, delete_config, destroy_backup, v):
"""
Easily back up installed packages, dotfiles, and more.
You can edit which dotfiles are backed up in ~/.shallow-backup.
Written by Aaron Lichtman (@alichtman).
"""
backup_config_path = get_config_path()
# No interface going to be displayed
if any([v, delete_config, destroy_backup, show, rm]) or None not in add:
if v:
print_version_info()
elif delete_config:
os.remove(backup_config_path)
print_bright_red("Removed config file...")
elif destroy_backup:
backup_home_path = get_config()["backup_path"]
destroy_backup_dir(backup_home_path)
elif None not in add:
add_path_to_config(add[0], add[1])
elif rm:
rm_path_from_config(rm)
elif show:
show_config()
sys.exit()
# Start CLI
splash_screen()
create_config_file_if_needed()
backup_config = get_config()
# User entered a new path, so update the config
if new_path:
abs_path = os.path.abspath(new_path)
print(Fore.BLUE + Style.NORMAL + "\nUpdating shallow-backup path to -> " + Style.BRIGHT + "{}".format(
abs_path) + Style.RESET_ALL)
backup_config["backup_path"] = abs_path
write_config(backup_config)
# User didn't enter any CLI args so prompt for path update before showing menu
elif not (old_path or complete or dotfiles or packages or fonts):
prompt_for_path_update(backup_config)
# Create backup directory and do git setup
backup_home_path = get_config()["backup_path"]
mkdir_warn_overwrite(backup_home_path)
repo, new_git_repo_created = safe_git_init(backup_home_path)
# Create default gitignore if we just ran git init
if new_git_repo_created:
safe_create_gitignore(backup_home_path)
# Prompt user for remote URL
if not remote:
prompt_for_git_url(repo)
# Set remote URL from CLI arg
if remote:
git_set_remote(repo, remote)
dotfiles_path = os.path.join(backup_home_path, "dotfiles")
configs_path = os.path.join(backup_home_path, "configs")
packages_path = os.path.join(backup_home_path, "packages")
fonts_path = os.path.join(backup_home_path, "fonts")
# Command line options
if any([complete, dotfiles, configs, packages, fonts, reinstall_packages, reinstall_configs]):
if reinstall_packages:
reinstall_packages_from_lists(packages_path)
elif reinstall_configs:
reinstall_config_files(configs_path)
elif complete:
backup_all(dotfiles_path, packages_path, fonts_path, configs_path)
git_add_all_commit_push(repo, "everything")
elif dotfiles:
backup_dotfiles(dotfiles_path)
git_add_all_commit_push(repo, "dotfiles")
elif configs:
backup_configs(configs_path)
git_add_all_commit_push(repo, "configs")
elif packages:
backup_packages(packages_path)
git_add_all_commit_push(repo, "packages")
elif fonts:
backup_fonts(fonts_path)
git_add_all_commit_push(repo, "fonts")
# No CL options, prompt for selection
else:
selection = actions_menu_prompt().lower().strip()
if selection == "back up everything":
backup_all(dotfiles_path, packages_path, fonts_path, configs_path)
git_add_all_commit_push(repo, "everything")
elif selection == "back up dotfiles":
backup_dotfiles(dotfiles_path)
git_add_all_commit_push(repo, "dotfiles")
elif selection == "back up configs":
backup_configs(configs_path)
git_add_all_commit_push(repo, "configs")
elif selection == "back up packages":
backup_packages(packages_path)
git_add_all_commit_push(repo, "packages")
elif selection == "back up fonts":
backup_fonts(fonts_path)
git_add_all_commit_push(repo, "fonts")
elif selection == "reinstall packages":
reinstall_packages_from_lists(packages_path)
elif selection == "reinstall configs":
reinstall_config_files(configs_path)
elif selection == "show config":
show_config()
elif selection == "destroy backup":
if prompt_yes_no("Erase backup directory: {}?".format(backup_home_path), Fore.RED):
destroy_backup_dir(backup_home_path)
else:
print_bright_red("Exiting to prevent accidental deletion of backup directory.")
sys.exit()
def cli(add, rm, show, all, dotfiles, configs, packages, fonts, old_path,
new_path, remote, reinstall_all, reinstall_configs, reinstall_dots,
reinstall_fonts, reinstall_packages, delete_config, destroy_backup, v):
"""
Easily back up installed packages, dotfiles, and more.
You can edit which dotfiles are backed up in ~/.shallow-backup.
Written by Aaron Lichtman (@alichtman).
"""
# Process CLI args
admin_action = any([v, delete_config, destroy_backup, show, rm
]) or None not in add
has_cli_arg = any([
old_path, all, dotfiles, packages, fonts, configs, reinstall_dots,
reinstall_fonts, reinstall_all, reinstall_configs, reinstall_packages
])
skip_prompt = any([
all, dotfiles, configs, packages, fonts, reinstall_packages,
reinstall_configs, reinstall_dots, reinstall_fonts
])
# Perform administrative action and exit.
if admin_action:
if v:
print_version_info()
elif delete_config:
# TODO: Error checking.
os.remove(get_config_path())
print_red_bold("Removed config file...")
elif destroy_backup:
backup_home_path = get_config()["backup_path"]
destroy_backup_dir(backup_home_path)
elif None not in add:
add_to_config(add[0], add[1])
elif rm:
rm_from_config(rm)
elif show:
show_config()
sys.exit()
# Start CLI
splash_screen()
safe_create_config()
backup_config = get_config()
# User entered a new path, so update the config
if new_path:
abs_path = os.path.abspath(new_path)
print(Fore.BLUE + Style.NORMAL +
"\nUpdating shallow-backup path to -> " + Style.BRIGHT +
"{}".format(abs_path) + Style.RESET_ALL)
backup_config["backup_path"] = abs_path
write_config(backup_config)
# User didn't enter any CLI args so prompt for path update before showing menu
elif not has_cli_arg:
prompt_for_path_update(backup_config)
# Create backup directory and do git setup
backup_home_path = get_config()["backup_path"]
mkdir_warn_overwrite(backup_home_path)
repo, new_git_repo_created = safe_git_init(backup_home_path)
# Create default gitignore if we just ran git init
if new_git_repo_created:
safe_create_gitignore(backup_home_path)
# Prompt user for remote URL
if not remote:
prompt_for_git_url(repo)
# Set remote URL from CLI arg
if remote:
git_set_remote(repo, remote)
dotfiles_path = os.path.join(backup_home_path, "dotfiles")
configs_path = os.path.join(backup_home_path, "configs")
packages_path = os.path.join(backup_home_path, "packages")
fonts_path = os.path.join(backup_home_path, "fonts")
# Command line options
if skip_prompt:
if reinstall_packages:
reinstall_packages_sb(packages_path)
elif reinstall_configs:
reinstall_configs_sb(configs_path)
elif reinstall_fonts:
reinstall_fonts_sb(fonts_path)
elif reinstall_dots:
reinstall_dots_sb(dotfiles_path)
elif reinstall_all:
reinstall_all_sb(dotfiles_path, packages_path, fonts_path,
configs_path)
elif all:
backup_all(dotfiles_path,
packages_path,
fonts_path,
configs_path,
skip=True)
git_add_all_commit_push(repo, "all")
elif dotfiles:
backup_dotfiles(dotfiles_path, skip=True)
git_add_all_commit_push(repo, "dotfiles")
elif configs:
backup_configs(configs_path, skip=True)
git_add_all_commit_push(repo, "configs")
elif packages:
backup_packages(packages_path, skip=True)
git_add_all_commit_push(repo, "packages")
elif fonts:
backup_fonts(fonts_path, skip=True)
git_add_all_commit_push(repo, "fonts")
# No CL options, show action menu and process selected option.
else:
selection = actions_menu_prompt().lower().strip()
selection_words = selection.split()
if selection.startswith("back up"):
if selection_words[-1] == "all":
backup_all(dotfiles_path, packages_path, fonts_path,
configs_path)
git_add_all_commit_push(repo, selection_words[-1])
elif selection_words[-1] == "dotfiles":
backup_dotfiles(dotfiles_path)
git_add_all_commit_push(repo, selection_words[-1])
elif selection_words[-1] == "configs":
backup_configs(configs_path)
git_add_all_commit_push(repo, selection_words[-1])
elif selection_words[-1] == "packages":
backup_packages(packages_path)
git_add_all_commit_push(repo, selection_words[-1])
elif selection_words[-1] == "fonts":
backup_fonts(fonts_path)
git_add_all_commit_push(repo, selection_words[-1])
elif selection.startswith("reinstall"):
if selection_words[-1] == "packages":
reinstall_packages_sb(packages_path)
elif selection_words[-1] == "configs":
reinstall_configs_sb(configs_path)
elif selection_words[-1] == "fonts":
reinstall_fonts_sb(fonts_path)
elif selection_words[-1] == "dotfiles":
reinstall_dots_sb(dotfiles_path)
elif selection_words[-1] == "all":
reinstall_all_sb(dotfiles_path, packages_path, fonts_path,
configs_path)
else:
if selection == "show config":
show_config()
elif selection == "destroy backup":
if prompt_yes_no(
"Erase backup directory: {}?".format(backup_home_path),
Fore.RED):
destroy_backup_dir(backup_home_path)
else:
print_red_bold(
"Exiting to prevent accidental deletion of backup directory."
)
sys.exit()
def parse_command_line(self, argv):
"""Parse the provided command line.
This sets up the default options for all apps. If an application
wants custom options, override the app_cmd_line_options method.
Arguments are parsed by the argparse module, and the resulting Namespace
object is available in self.args.
Arguments:
argv -- An array of parameters to parse, not including program name.
"""
parser = argparse.ArgumentParser(add_help=False)
LOG_LEVELS = ['TRACE', 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'FATAL']
parser.add_argument('-h', '--help', action='store_true',
help='show this help message and exit')
parser.add_argument('--log-level', dest="log_level",
help='Log level', choices = LOG_LEVELS)
parser.add_argument('--log-method', dest="log_method",
help='Where to log', choices = ['console', 'syslog'])
facilities = SysLogHandler.facility_names.keys()
facilities.sort()
parser.add_argument('--log-syslog-facility', dest="syslog_facility",
help='Which syslog facility to log to.',
choices = facilities)
parser.add_argument('-d', '--debug', action='store_true',
help='Shortcut to set console logging with level DEBUG')
parser.add_argument('-t', '--trace', action='store_true',
help='Shortcut to set console logging with level TRACE')
parser.add_argument('--config-dir', dest="config_dir",
help='Directory with configuration files')
parser.add_argument('--state-dir', dest="state_dir",
help='Directory with local state files')
self.app_cmd_line_options(parser)
# If parsing fails, this will print error and exit
args = parser.parse_args(argv[1:])
if args.config_dir:
config.set_config_dir(args.config_dir)
if args.state_dir:
config.set_localstate_dir(args.state_dir)
if args.help:
config.init_directories()
parser.print_help()
print
print "Paths"
print " Default config dir: %s" % _directories.DEFAULT_CONFDIR
print " Default state dir : %s" % _directories.DEFAULT_LOCALSTATEDIR
print " Active config dir : %s" % config.CONFDIR
print " Active state dir : %s" % config.LOCALSTATEDIR
print
print "System configuration file : %s" % config.get_config_path('conf.d/system.conf')
print "App-specific configuration file: %s" % config.get_config_path('conf.d/%s.conf'% self.app_short_name)
print
return False
self.args = args
return True
def __init__(self, parent=None):
log_conf = get_logging_config()
QSplitter.__init__(self, parent)
self.setWindowTitle(MAIN_WINDOW_TITLE)
self.setWindowIcon(get_icon("agent.svg"))
self.sysTray = SystemTray(self)
self.connect(self.sysTray,
SIGNAL("activated(QSystemTrayIcon::ActivationReason)"),
self.__icon_activated)
checks = get_checks()
datadog_conf = DatadogConf(get_config_path())
self.create_logs_files_windows(log_conf)
listwidget = QListWidget(self)
listwidget.addItems([
osp.basename(check.module_name).replace("_", " ").title()
for check in checks
])
self.properties = PropertiesWidget(self)
self.setting_button = QPushButton(get_icon("info.png"),
"Logs and Status", self)
self.menu_button = QPushButton(get_icon("settings.png"), "Actions",
self)
self.settings = [
("Forwarder Logs", lambda: [
self.properties.set_log_file(self.forwarder_log_file),
self.show_html(self.properties.group_code, self.properties.
html_window, False)
]),
("Collector Logs", lambda: [
self.properties.set_log_file(self.collector_log_file),
self.show_html(self.properties.group_code, self.properties.
html_window, False)
]),
("Dogstatsd Logs", lambda: [
self.properties.set_log_file(self.dogstatsd_log_file),
self.show_html(self.properties.group_code, self.properties.
html_window, False)
]),
("JMX Fetch Logs", lambda: [
self.properties.set_log_file(self.jmxfetch_log_file),
self.show_html(self.properties.group_code, self.properties.
html_window, False)
]),
]
if Platform.is_windows():
self.settings.extend([
("Service Logs", lambda: [
self.properties.set_log_file(self.service_log_file),
self.show_html(self.properties.group_code, self.properties.
html_window, False)
]),
])
self.settings.extend([
("Agent Status", lambda: [
self.properties.html_window.setHtml(self.properties.html_window
.latest_status()),
self.show_html(self.properties.group_code, self.properties.
html_window, True),
self.properties.set_status()
]),
])
self.agent_settings = QPushButton(get_icon("edit.png"), "Settings",
self)
self.connect(
self.agent_settings, SIGNAL("clicked()"), lambda: [
self.properties.set_datadog_conf(datadog_conf),
self.show_html(self.properties.group_code, self.properties.
html_window, False)
])
self.setting_menu = SettingMenu(self.settings)
self.connect(
self.setting_button, SIGNAL("clicked()"),
lambda: self.setting_menu.popup(
self.setting_button.mapToGlobal(QPoint(0, 0))))
self.manager_menu = Menu(self)
self.connect(
self.menu_button, SIGNAL("clicked()"),
lambda: self.manager_menu.popup(
self.menu_button.mapToGlobal(QPoint(0, 0))))
holdingBox = QGroupBox("", self)
Box = QVBoxLayout(self)
Box.addWidget(self.agent_settings)
Box.addWidget(self.setting_button)
Box.addWidget(self.menu_button)
Box.addWidget(listwidget)
holdingBox.setLayout(Box)
self.addWidget(holdingBox)
self.addWidget(self.properties)
self.connect(self.properties.enable_button, SIGNAL("clicked()"),
lambda: enable_check(self.properties))
self.connect(self.properties.disable_button, SIGNAL("clicked()"),
lambda: disable_check(self.properties))
self.connect(self.properties.save_button, SIGNAL("clicked()"),
lambda: save_file(self.properties))
self.connect(
self.properties.refresh_button, SIGNAL("clicked()"), lambda: [
self.properties.set_log_file(self.properties.current_file),
self.properties.html_window.setHtml(self.properties.html_window
.latest_status())
])
self.connect(
listwidget, SIGNAL('currentRowChanged(int)'), lambda row: [
self.properties.set_item(checks[row]),
self.show_html(self.properties.group_code, self.properties.
html_window, False)
])
listwidget.setCurrentRow(0)
self.setSizes([150, 1])
self.setStretchFactor(1, 1)
self.resize(QSize(950, 600))
self.properties.set_datadog_conf(datadog_conf)
self.do_refresh()
import json
import image_fetch_utils as ifu
import image_fetch_dataset as ifd
from config import get_config_path
with open(get_config_path()) as cfg_file:
cfg_data = json.load(cfg_file)
out_path = cfg_data['file_paths']['output_path']
test_img_path = cfg_data['file_paths']['test_image_path']
train_img_path = cfg_data['file_paths']['train_image_path']
mat_path = cfg_data['file_paths']['mat_path']
csv_path = cfg_data['file_paths']['csv_path']
vgd, potentials = None, None
platt_mod, bin_mod, queries = None, None, None
ifdata, hf = None, None
data_loaded = None
def _load(dataset='stanford', split='test', use_csv=False):
"""Helper function which loads data and caches it for later use."""
global data_loaded, vgd, potentials, platt_mod, bin_mod, queries, ifdata
if (dataset, split) != data_loaded:
data = ifu.get_mat_data(mat_path,
get_potentials=not use_csv,
get_bin_mod=not use_csv,
get_platt_mod=not use_csv)
vgd, potentials, platt_mod, bin_mod, queries = data
img_path = test_img_path if split == 'test' else train_img_path
if use_csv:
