def get_config_value(configuration_file, option, verbose=False): # mongodb config files do not have sections, so inject a dummy section # see: https://stackoverflow.com/questions/2819696/parsing-properties-file-in-python/ config = io.StringIO() config.write('[dbdat]\n') config.write(open(configuration_file).read()) config.seek(0, os.SEEK_SET) value = None try: configuration = configparser.configparser() configuration.readfp(config) except configparser.ParsingError as e: if verbose: print('Ignoring parsing errors:\n' + str(e)) try: value = configuration.get('dbdat', option) # clean up required value = value.split()[0] value = value.translate(None, "'") except configparser.NoOptionError as e: value = None return value
def load_config(config_file_name): config = configparser() try: with open(config_file_name, 'r') as w: #sanity check because configparser is stupid config.read(config_file_name) print("Config loaded succesfully!") except: print( "Please add your token and other information to winbot.ini, then run the script again." ) config['winbot'] = {} config['winbot']['token'] = 'SetMeUp' config['winbot']['prefix'] = 'vm!' config['winbot']['vm_name'] = 'Example' config['winbot']['owner_id'] = '12345' config['winbot']['channel_id'] = '67890' with open(config_file_name, 'w') as f: config.write(f) sys.exit(1) if config['winbot']['token'] == 'SetMeUp' or len( config['winbot']['token']) < 8: print("winbot.ini contains invalid data. Winbot will now terminate.") sys.exit(1) else: return_config = [] return_config.append(config['winbot']['token']) return_config.append(config['winbot']['prefix']) return_config.append(config['winbot']['vm_name']) return_config.append(int(config['winbot']['owner_id'])) return_config.append(int(config['winbot']['channel_id'])) return return_config
def __init__(self, path, force=False): ''' init worktree and git dir read Configuration file: a INI-like file with a single section ([core]) + three fields. ''' self.worktree = path self.gitdir = os.path.join(path, '.git') if not (force or os.path.isdir(self.gitdir)): raise Exception('{0:s} is not a Git repository'.format( self.worktree)) self.conf = configparser.configparser() # wait? cf = repo_file(self, 'config') if cf and os.path.exists(cf): self.config.Read([cf]) elif not force: raise Exception('Configuration file missing.') if not force: vers = int(self.conf.get('core', 'repositoryformatversion')) if vers != 0: raise Exception( 'Unsupported repositoryformatversion: {0:s}'.format(vers))
def do_check(self, configuration_file): configuration = configparser.configparser() try: configuration.read(configuration_file) except configparser.ParsingError as e: if self.verbose: print('Ignoring parsing errors:\n' + str(e)) try: general_log_file = configuration.get('mysqld', 'log_error') self.result['level'] = 'GREEN' self.result['output'] = 'Error log file enabled.' log_warnings = configuration.get('mysqld', 'log_warnings') if log_warnings < 2: self.result['level'] = 'RED' self.result[ 'output'] = 'Error log not capturing access denied events.' except configparser.NoOptionError as e: self.result['level'] = 'RED' self.result['output'] = 'Error log file not enabled.' return self.result
def mark(self, id, duedate, description, completed, done=None, delete=None): username = logon.checkauth() taskdir = gettaskdir(username) try: int( id, 16 ) # check whether id is plain hex integer without nastiness like .. or / except ValueError: raise cherrypy.InternalRedirect(self.logoffpath) filename = os.path.join(taskdir, id + '.task') if done == "Done": d = configparser() with open(filename, "r") as file: d.readfp(file) if completed == "" or completed == "None": completed = date.today().isoformat() d.set('task', 'completed', completed) with open(filename, "w") as file: d.write(file) elif delete == "Del": os.unlink(filename) raise cherrypy.InternalRedirect(".")
def index(self): username = logon.checkauth() taskdir = gettaskdir(username) tasklist = glob.glob(os.path.join(taskdir,'*.task')) tasks = ['<div class="header">Tasklist for user : <span class="highlight">%s</span><form class="logoff" action="%s" method="GET"><button type="submit" name="logoffurl" class="logoff-button" value="/">Log off</button></form></div>'%(username,self.logoffpath), '<div class="taskheader"><div class="left">Due date</div><div class="middle">Description</div><div class="right">Completed</div></div>', '<div id="items" class="ui-widget-content">'] for filename in tasklist: d = configparser(defaults={'description':'','duedate':'','completed':None}) id = os.path.splitext(os.path.basename(filename))[0] d.readfp(open(filename)) description = d.get('task','description') duedate = d.get('task','duedate') completed = d.get('task','completed') tasks.append('''<form class="%s" action="mark" method="GET"> <input type="text" class="duedate left" name="duedate" value="%s" readonly="readonly" /> <input type="text" class="description middle" name="description" value="%s" readonly="readonly" /> <input type="text" class="completed right editable-date tooltip" title="click to select a date, then click done" name="completed" value="%s" /> <input type="hidden" name="id" value="%s" /> <button type="submit" class="done-button" name="done" value="Done" >Done</button> <button type="submit" class="del-button" name="delete" value="Del" >Del</button> </form> '''%('notdone' if completed==None else 'done',duedate,description,completed,id)) tasks.append('''<form class="add" action="add" method="GET"> <input type="text" class="duedate left editable-date tooltip" name="duedate" title="click for a date" /> <input type="text" class="description middle tooltip" title="click to enter a description" name="description"/> <button type="submit" class="add-button" name="add" value="Add" >Add</button> </form></div> ''') return base_page%('itemlist',"".join(tasks))
def get_redis_config(): config_path = '\\'.join([os.getcwd(), "\conf\config.properties"]) config = configparser.configparser() config.read(config_path) host = config.get('redis', 'host') port = int(config.get('redis', 'port')) return host, port
def do_check(self, configuration_file): configuration = configparser.configparser() try: configuration.read(configuration_file) except configparser.ParsingError as e: if self.verbose: print('Ignoring parsing errors:\n' + str(e)) try: bind_address = configuration.get('httpd', 'bind_address') if '127.0.0.1' == bind_address or 'localhost' == bind_address: self.result['level'] = 'GREEN' self.result['output'] = 'Database listening on localhost only. (' + str(bind_address) + ')' else: self.result['level'] = 'YELLOW' self.result['output'] = 'Database listening is not localhost only (' + str(bind_address) + ')' except configparser.NoOptionError as e: self.result['level'] = 'GREEN' self.result['output'] = 'bind-address option not set, default is 127.0.0.1 or localhost.' return self.result
def get_statsd_config(): config_path = '\\'.join([os.getcwd(), "\conf\config.properties"]) config = configparser.configparser() config.read(config_path) host = config.get('statsd', 'host') port = int(config.get('statsd', 'port')) prefix = config.get('statsd', 'prefix') return host, port, prefix
def __init__(self, configfile = "cluster.ini"): self.log = ACSLog("ACSUtils") self.log.debug("Reading config from " + configfile) defaults = {"orchestratorType": "Mesos"} config = configparser.configparser(defaults) config.read(configfile) if not config.has_option('Group', 'name'): config.set('Group', 'name', config.get('ACS', 'dnsPrefix')) self.config = config
def __init__(self): self.config = configparser() x = path_join(getcwd(), 'periastron.conf') for filepath in ['etc/periastron.conf', '/etc/periastron/periastron.conf', '../etc/periastron.conf', x]: if isfile(filepath): print('Using config file {}'.format(filepath)) self.config.read(filepath) break
def add(self,add,description,duedate): username = logon.checkauth() taskdir = gettaskdir(username) filename = os.path.join(taskdir,uuid().hex+'.task') d=configparser() d.add_section('task') d.set('task','description',description) d.set('task','duedate',duedate) with open(filename,"w") as file: d.write(file) raise cherrypy.InternalRedirect(".")
def parseParams_v1(fname): #print(fname) paramsDict = {} params = configparser.configparser() params.read(fname) for section in params.sections(): paramsDict[section] = {} for datum in params.options(section): try: paramsDict[section].update(paramHandler_v1[section](datum, params.get(section, datum).split(' '))) except KeyError: raise KeyError('%s not a valid category. Valid categories are: %s' % (section, str(paramHandler_v1.keys()))) paramsDict['Static AAs'] = [entry[1] for entry in paramsDict['Static Mods']] return paramsDict
def mark(self,id,duedate,description,completed,done=None,delete=None): username = logon.checkauth() taskdir = gettaskdir(username) filename = os.path.join(taskdir,id+'.task') if done=="Done": print('####',id,duedate,description,completed,done,delete) d=configparser() with open(filename,"r") as file: d.readfp(file) if completed == "" or completed == "None": completed = date.today().isoformat() d.set('task','completed',completed) with open(filename,"w") as file: d.write(file) elif delete=="Del": os.unlink(filename) raise cherrypy.InternalRedirect(".")
def read_worker_state(): # Reads the state.config file and returns the values of pid, workspace_id, resource_running_version if os.path.isfile(WORKER_STATE_FILE_PATH): state = configparser.configparser() try: state.read(WORKER_STATE_FILE_PATH) pid = state.get(STATE_SECTION, PID) workspace_id = state.get(STATE_SECTION, WORKSPACE_ID) resource_running_version = state.get(STATE_SECTION, DSC_RESOURCE_VERSION) except configparser.NoSectionError, exception: log(DEBUG, exception.message) raise configparser.Error(exception.message) except configparser.NoOptionError, exception: log(DEBUG, exception.message) raise configparser.Error(exception.message)
def do_check(self, configuration_file): configuration = configparser.configparser() count = 0 try: configuration.read(configuration_file) except configparser.ParsingError as e: if self.verbose: print('Ignoring parsing errors:\n' + str(e)) try: error_log_file = configuration.get('mysqld', 'log_error')[0] if os.path.isfile(str(error_log_file)): with open(str(error_log_file), 'r') as log: for line in log: if "Access denied for user '" + self.appuser + "'" in line: count += 1 if 0 == count: self.result['level'] = 'GREEN' self.result[ 'output'] = 'No access denied events found for user.' elif count < 5: self.result['level'] = 'YELLOW' self.result[ 'output'] = 'Access denied events found for user.' else: self.result['level'] = 'RED' self.result[ 'output'] = 'Excessive access denied events found for user.' else: self.result['level'] = 'YELLOW' self.result[ 'output'] = 'Could not access error log file ' + str( error_log_file) + '.' except configparser.NoOptionError as e: self.result['level'] = 'RED' self.result['output'] = 'Error log file not enabled.' return self.result
def do_check(self, configuration_file): configuration = configparser.configparser() try: configuration.read(configuration_file) except configparser.ParsingError as e: if self.verbose: print('Ignoring parsing errors:\n' + str(e)) try: general_log_file = configuration.get('mysqld', 'general_log_file') except configparser.NoOptionError as e: self.result['level'] = 'YELLOW' self.result['output'] = 'General log file not enabled.' return self.result
def mark(self,id,duedate,description,completed,done=None,delete=None): username = logon.checkauth() taskdir = gettaskdir(username) try: int(id,16) # check whether id is plain hex integer without nastiness like .. or / except ValueError: raise cherrypy.InternalRedirect(self.logoffpath) filename = os.path.join(taskdir,id+'.task') if done=="Done": d=configparser() with open(filename,"r") as file: d.readfp(file) if completed == "" or completed == "None": completed = date.today().isoformat() d.set('task','completed',completed) with open(filename,"w") as file: d.write(file) elif delete=="Del": os.unlink(filename) raise cherrypy.InternalRedirect(".")
def main(argv): global username, password, host, device_id try: opts, args = getopt.getopt(argv,"hc:",["config="]) except getopt.GetoptError: help() for opt, arg in opts: if opt == '-c': cfg = arg config = configparser() config.read_file(open(cfg)) username = config['dysondata']['username'] password = config['dysondata']['password'] host = config['dysondata']['host'] device_id = config['dysondata']['device_id'] port = config.getint('dysondata','port') interval = config.getint('dysondata','interval') start_client(host, username, password, port, interval) else: help()
def do_check(self, configuration_file): configuration = configparser.configparser() try: configuration.read(configuration_file) except configparser.ParsingError as e: if self.verbose: print('Ignoring parsing errors:\n' + str(e)) try: general_log_file = configuration.get('client', 'password') # if the option is found, then red! self.result['level'] = 'RED' self.result['output'] = 'Client password is in use.' except configparser.NoOptionError as e: self.result['level'] = 'GREEN' self.result['output'] = 'Client password not used.' return self.result
def login(user,password): ''' 登录功能。 验证密码三次错误或用户名输入错误直接返回0 登录成功返回1并将文件中登录次数置写0 密码错误返回2并记录登录次数 ''' cf = configparser.configparser() cf.read("simulateDB.cfg") jusrinfo = cf.get('user','usrinfo') usrinfo = json.loads(jusrinfo) authority = 0 role = 0 if usrinfo['user'] == user and usrinfo['times'] < 3: if usrinfo['password'] == password: usrinfo['times'] = 0 flag = 1 else: usrinfo['times'] += 1 flag = 2 else: flag = 0 if flag: jcardfloatinfo = json.dumps(usrinfo) cf.set('user','usrinfo',jcardfloatinfo) cf.write(open('simulateDB.cfg', "w")) if flag == 1: jroleinfo = cf.get('role','roleinfo') print(jroleinfo) roleinfo = json.loads(jroleinfo) for key in roleinfo: if user in roleinfo[key]: role = key print(role) authority = cf.get('authority',role) break return str(flag),role,authority
import os, configparser #Check if .pyscholar exists and create it if necessary # chmod(0777) pyscholarDir = os.path.join(os.path.expanduser("~user"), "pyscholar") if not os.path.exists(pyscholarDir): os.makedirs(pyscholarDir) #Same for .pyscholar/keys.cfg if not os.path.exists(os.path.join(pyscholarDir, "keys.cfg")): keysParser = configparser.configparser() keysParser.add_section("Keys") keysParser.set('Keys', 'Scopus', "") originalMask = os.umask(0) keysDescriptor = os.open(os.path.join(pyscholarDir,"keys.cfg"),os.O_WRONLY | os.O_CREAT) keysFile = os.fdopen(keysDescriptor, 'w') os.umask(originalMask) keysParser.write(keysFile) keysFile.close() #import scopus #import network
def parse_config(self, argv): def join_conf_dir(path): # Turn relative paths to absolute paths, depending on the # location of the INI (or rather CONF_DIR which by its definition # is the location of the INI). return path if os.path.isabs(path) else os.path.join( CONF_DIR, path) config = configparser() if len(argv) == 1: config_file = "my-site" else: config_file = argv[1] config_path = os.path.join(CONF_DIR, config_file + '.ini') logger.info("Reading %s", config_path) config.read(config_path) self.config = {} try: self.config['server'] = {} self.config['server']['name'] = config_file self.config['server']['loglevel'] = int( config['server']['loglevel']) self.config['server']['host'] = config['server']['host'] self.config['server']['port'] = int(config['server']['port']) self.config['server']['enable_mail'] = config['server'][ 'enable_mail'] self.config['server']['repo_dir'] = join_conf_dir( config['server']['repo_dir']) self.config['server']['temp_repo_dir'] = join_conf_dir( config['server']['temp_repo_dir']) self.config['server']['valid_languages'] = config['server'][ 'valid_languages'] if config['server']['max_threads'] == 'max': self.config['server'][ 'max_threads'] = multiprocessing.cpu_count() else: self.config['server']['max_threads'] = int( config['server']['max_threads']) self.config['targets'] = {} for section in config.sections(): if not str(section).startswith("target_"): continue self.config['targets'][config[section]['name']] = {} self.config['targets'][ config[section]['name']]['name'] = config[section]['name'] self.config['targets'][ config[section]['name']]['template_dir'] = join_conf_dir( config[section]['template_dir']) self.config['targets'][config[section]['name']][ 'active'] = config[section]['active'] self.config['targets'][config[section]['name']][ 'draft'] = config[section]['draft'] self.config['targets'][config[section]['name']][ 'remarks'] = config[section]['remarks'] self.config['targets'][ config[section]['name']]['meta'] = config[section]['meta'] self.config['targets'][config[section]['name']][ 'default_xslt_params'] = join_conf_dir( config[section]['default_xslt_params']) self.config['targets'][ config[section]['name']]['enable_target_sync'] = config[ section]['enable_target_sync'] if config[section]['enable_target_sync'] == 'yes': self.config['targets'][config[section]['name']][ 'target_path'] = config[section]['target_path'] self.config['targets'][ config[section]['name']]['backup_path'] = join_conf_dir( config[section]['backup_path']) self.config['targets'][config[section] ['name']]['config_dir'] = join_conf_dir( config[section]['config_dir']) self.config['targets'][config[section]['name']][ 'languages'] = config[section]['languages'] self.config['targets'][config[section]['name']][ 'default_lang'] = config[section]['default_lang'] self.config['targets'][config[section]['name']][ 'omit_default_lang_path'] = config[section][ 'omit_default_lang_path'] self.config['targets'][config[section]['name']][ 'internal'] = config[section]['internal'] self.config['targets'][config[section]['name']][ 'zip_formats'] = config[section]['zip_formats'] self.config['targets'][config[section]['name']][ 'server_base_path'] = config[section]['server_base_path'] self.config['targets'][ config[section]['name']]['canonical_url_domain'] = config[ section]['canonical_url_domain'] self.config['targets'][config[section]['name']][ 'server_root_files'] = join_conf_dir( config[section]['server_root_files']) self.config['targets'][ config[section]['name']]['enable_ssi_fragments'] = config[ section]['enable_ssi_fragments'] if config[section]['enable_ssi_fragments'] == 'yes': self.config['targets'][config[section]['name']][ 'fragment_dir'] = join_conf_dir( config[section]['fragment_dir']) self.config['targets'][config[section]['name']][ 'fragment_l10n_dir'] = join_conf_dir( config[section]['fragment_l10n_dir']) # FIXME: I guess this is not the prettiest way to handle # optional values (but it works for now) self.config['targets'][config[section] ['name']]['build_container'] = False if 'build_container' in list(config[section].keys()): self.config['targets'][config[section]['name']][ 'build_container'] = config[section]['build_container'] self.config['targets'][config[section]['name']][ 'site_sections'] = config[section]['site_sections'] self.config['targets'][ config[section]['name']]['default_site_section'] = config[ section]['default_site_section'] except KeyError as error: logger.warning( "Invalid configuration file, missing configuration key %s. Exiting.", error) sys.exit(1)
def parse_config(self, argv): config = configparser() if len(argv) == 1: config_file = "docserv" else: config_file = argv[1] config_path = os.path.join(CONF_DIR, config_file + '.ini') logger.info("Reading %s", config_path) config.read(config_path) self.config = {} try: self.config['server'] = {} self.config['server']['name'] = config_file self.config['server']['loglevel'] = int( config['server']['loglevel']) self.config['server']['host'] = config['server']['host'] self.config['server']['port'] = int(config['server']['port']) self.config['server']['repo_dir'] = config['server']['repo_dir'] self.config['server']['temp_repo_dir'] = config['server'][ 'temp_repo_dir'] self.config['server']['valid_languages'] = config['server'][ 'valid_languages'] self.config['server']['max_threads'] = int( config['server']['max_threads']) self.config['targets'] = {} for section in config.sections(): if not str(section).startswith("target_"): continue self.config['targets'][config[section]['name']] = {} self.config['targets'][ config[section]['name']]['name'] = config[section]['name'] self.config['targets'][config[section]['name']][ 'template_dir'] = config[section]['template_dir'] self.config['targets'][config[section]['name']][ 'active'] = config[section]['active'] self.config['targets'][config[section]['name']][ 'draft'] = config[section]['draft'] self.config['targets'][config[section]['name']][ 'remarks'] = config[section]['remarks'] self.config['targets'][ config[section]['name']]['meta'] = config[section]['meta'] self.config['targets'][ config[section]['name']]['default_xslt_params'] = config[ section]['default_xslt_params'] self.config['targets'][config[section]['name']][ 'target_path'] = config[section]['target_path'] self.config['targets'][config[section]['name']][ 'backup_path'] = config[section]['backup_path'] self.config['targets'][config[section]['name']][ 'config_dir'] = config[section]['config_dir'] self.config['targets'][config[section]['name']][ 'languages'] = config[section]['languages'] self.config['targets'][config[section]['name']][ 'default_lang'] = config[section]['default_lang'] self.config['targets'][config[section]['name']][ 'omit_default_lang_path'] = config[section][ 'omit_default_lang_path'] self.config['targets'][config[section]['name']][ 'internal'] = config[section]['internal'] self.config['targets'][config[section]['name']][ 'zip_formats'] = config[section]['zip_formats'] self.config['targets'][config[section]['name']][ 'server_base_path'] = config[section]['server_base_path'] self.config['targets'][ config[section]['name']]['canonical_url_domain'] = config[ section]['canonical_url_domain'] self.config['targets'][config[section]['name']][ 'htaccess'] = config[section]['htaccess'] self.config['targets'][config[section]['name']][ 'favicon'] = config[section]['favicon'] except KeyError as error: logger.warning( "Invalid configuration file, missing configuration key '%s'. Exiting.", error) sys.exit(1)
def main(): # set up command line argument processing parser = argparse.ArgumentParser() # options parser.add_argument( "-v", "--verbose", help="increase output verbosity", action="store_true", default=False, ) parser.add_argument( "-n", "--dry-run", help="Process data but don't save results", action="store_true", default=False, ) parser.add_argument( "-p", "--aggregation-period", help="Number of Days to Aggregate (default=1)", nargs="?", type=int, choices=range(1, 5, 2), default=1, ) # positional arguments parser.add_argument("site", help="PhenoCam site name") parser.add_argument("roiname", help="ROI name, e.g. canopy_0001") # get args args = parser.parse_args() sitename = args.site roiname = args.roiname verbose = args.verbose dryrun = args.dry_run ndays = args.aggregation_period if verbose: print("site: {0}".format(sitename)) print("roiname: {0}".format(roiname)) print("verbose: {0}".format(verbose)) print("dryrun: {0}".format(dryrun)) print("period: {0}".format(ndays)) # read in config file for this ROI List if it exists config_file = "{0}_{1}.cfg".format(sitename, roiname) config_path = os.path.join(archive_dir, sitename, "ROI", config_file) if os.path.exists(config_path): # NOTE: should probably subclass safe config parser # and add gettime() method which checks for time validity cfgparser = configparser( defaults={ "nimage_threshold": str(default_nimage_threshold), "time_min": str(default_time_min), "time_max": str(default_time_max), "sunelev_min": str(default_sunelev_min), "brt_min": str(default_brt_min), "brt_max": str(default_brt_max), } ) cfgparser.read(config_path) if cfgparser.has_section("gcc90_calculation"): nimage_threshold = cfgparser.getint("gcc90_calculation", "nimage_threshold") time_max_str = cfgparser.get("gcc90_calculation", "time_max") [tmax_hr, tmax_mn, tmax_sc] = time_max_str.split(":") time_max = time(int(tmax_hr), int(tmax_mn), int(tmax_sc)) time_min_str = cfgparser.get("gcc90_calculation", "time_min") [tmin_hr, tmin_mn, tmin_sc] = time_min_str.split(":") time_min = time(int(tmin_hr), int(tmin_mn), int(tmin_sc)) sunelev_min = cfgparser.getfloat("gcc90_calculation", "sunelev_min") brt_min = cfgparser.getint("gcc90_calculation", "brt_min") brt_max = cfgparser.getint("gcc90_calculation", "brt_max") else: nimage_threshold = int(default_nimage_threshold) [tmax_hr, tmax_mn, tmax_sc] = default_time_max.split(":") time_max = time(int(tmax_hr), int(tmax_mn), int(tmax_sc)) [tmin_hr, tmin_mn, tmin_sc] = default_time_min.split(":") time_min = time(int(tmin_hr), int(tmin_mn), int(tmin_sc)) sunelev_min = default_sunelev_min brt_min = default_brt_min brt_max = default_brt_max else: nimage_threshold = int(default_nimage_threshold) [tmax_hr, tmax_mn, tmax_sc] = default_time_max.split(":") time_max = time(int(tmax_hr), int(tmax_mn), int(tmax_sc)) [tmin_hr, tmin_mn, tmin_sc] = default_time_min.split(":") time_min = time(int(tmin_hr), int(tmin_mn), int(tmin_sc)) sunelev_min = default_sunelev_min brt_min = default_brt_min brt_max = default_brt_max # print config values if verbose: print("") print("gcc config:") print("===========") print("roi_list: ", "{0}_{1}_roi.csv".format(sitename, roiname)) if os.path.exists(config_path): print("config file: {0}".format(config_file)) else: print("config file: None") print("nimage threshold: ", nimage_threshold) print("time of day min: ", time_min) print("time of day max: ", time_max) print("sun elev min: ", sunelev_min) print("aggregate days: ", ndays) print("minimum brightness: ", brt_min) print("maximum brightness: ", brt_max) # set up output filename outdir = os.path.join(archive_dir, sitename, "ROI") outfile = "{0}_{1}_{2}day.csv".format(sitename, roiname, ndays) outpath = os.path.join(outdir, outfile) if verbose: print("output file: ", outfile) # create gcc timeseries object as empty list gcc_ts = GCCTimeSeries( site=sitename, ROIListID=roiname, nday=ndays, nmin=nimage_threshold, tod_min=time_min, tod_max=time_max, sunelev_min=sunelev_min, brt_min=brt_min, brt_max=brt_max, ) # get roi timeseries for this site and roi roits = get_roi_timeseries(sitename, roiname) if verbose: print("") print("ROI timeseries info:") print("====================") print("site: ", roits.site) print("ROI list id: ", roits.roilistid) print("create date: ", roits.created_at) print("update date: ", roits.updated_at) print("nrows: ", len(roits.rows)) # make list of rows which match image selection criteria roits_rows = roits.select_rows( tod_min=time_min, tod_max=time_max, sunelev_min=sunelev_min, brt_min=brt_min, brt_max=brt_max, ) # check that some rows passed selection criteria nrows = len(roits_rows) if nrows == 0: print("No rows passed the selection criteria") return if verbose: print("Number of selected rows: {0}".format(nrows)) # make a list of dates for selected images img_date = [] for row in roits_rows: img_date.append(row["datetime"].date()) # list is ordered so find first and last dates dt_first = img_date[0] dt_last = img_date[nrows - 1] if verbose: print("date first: {}".format(dt_first)) print("date last: {}".format(dt_last)) # set up a generator which yields dates for the start # of the next nday period covering the date range of image gcc_dr = daterange2(dt_first, dt_last, ndays) # calculate offset for timeseries based on nday day_offset = ndays / 2 date_offset = timedelta(days=day_offset) # roits_ndx will be index into ROI timeseries roits_ndx = 0 # set up vars for accumulating stats img_cnt = 0 filenames = [] r_dn_vals = [] rcc_vals = [] g_dn_vals = [] gcc_vals = [] b_dn_vals = [] bcc_vals = [] solar_elev_vals = [] midday_delta_vals = [] # loop over nday time periods for gcc_ndx, start_date in enumerate(gcc_dr): end_date = start_date + timedelta(ndays) gcc_date = start_date + date_offset doy = gcc_date.timetuple().tm_yday midday_noon = datetime(gcc_date.year, gcc_date.month, gcc_date.day, 12, 0, 0) # get roits rows for this time period while ( roits_ndx < nrows and img_date[roits_ndx] >= start_date and img_date[roits_ndx] < end_date ): # skip this row if awbflag is 1 if roits_rows[roits_ndx]["awbflag"] == 1: if roits_ndx < nrows: roits_ndx += 1 continue else: break filenames.append(roits_rows[roits_ndx]["filename"]) r_dn = roits_rows[roits_ndx]["r_mean"] r_dn_vals.append(r_dn) g_dn = roits_rows[roits_ndx]["g_mean"] g_dn_vals.append(g_dn) b_dn = roits_rows[roits_ndx]["b_mean"] b_dn_vals.append(b_dn) dnsum = r_dn + g_dn + b_dn # NOTE: I'm recomputing gcc, rcc, bcc from DN values rather # than using value stored in roistats CSV if dnsum <= 0: rcc = np.nan bcc = np.nan gcc = np.nan else: img_cnt += 1 rcc = r_dn / dnsum bcc = b_dn / dnsum gcc = roits_rows[roits_ndx]["gcc"] solar_elev = roits_rows[roits_ndx]["solar_elev"] # note that rcc_vals can include NaN's rcc_vals.append(rcc) gcc_vals.append(gcc) bcc_vals.append(bcc) solar_elev_vals.append(solar_elev) midday_td = roits_rows[roits_ndx]["datetime"] - midday_noon midday_td_secs = np.abs(midday_td.days * 86400 + midday_td.seconds) midday_delta_vals.append(midday_td_secs) if roits_ndx < nrows: roits_ndx += 1 else: break # check to see if we got any (good) images if img_cnt == 0: # nodata for this time period image_count = 0 midday_filename = ND_STRING midday_r = ND_FLOAT midday_g = ND_FLOAT midday_b = ND_FLOAT midday_gcc = ND_FLOAT midday_rcc = ND_FLOAT r_mean = ND_FLOAT r_std = ND_FLOAT g_mean = ND_FLOAT g_std = ND_FLOAT b_mean = ND_FLOAT b_std = ND_FLOAT gcc_mean = ND_FLOAT gcc_std = ND_FLOAT gcc_50 = ND_FLOAT gcc_75 = ND_FLOAT gcc_90 = ND_FLOAT rcc_mean = ND_FLOAT rcc_std = ND_FLOAT rcc_50 = ND_FLOAT rcc_75 = ND_FLOAT rcc_90 = ND_FLOAT max_solar_elev = ND_FLOAT snow_flag = ND_INT outlierflag_gcc_mean = ND_INT outlierflag_gcc_50 = ND_INT outlierflag_gcc_75 = ND_INT outlierflag_gcc_90 = ND_INT # got some good images but not enough - probably there # are cases where this will fail e.g. not images on the # midday of a 3-day aggregation period. elif img_cnt < nimage_threshold: # not enough images image_count = img_cnt # find nearest image to midday (noon) on mid-interval date mi_ndx = midday_delta_vals.index(min(midday_delta_vals)) midday_filename = filenames[mi_ndx] midday_r = r_dn_vals[mi_ndx] midday_g = g_dn_vals[mi_ndx] midday_b = b_dn_vals[mi_ndx] midday_gcc = gcc_vals[mi_ndx] midday_rcc = rcc_vals[mi_ndx] # no stats for this time interval r_mean = ND_FLOAT r_std = ND_FLOAT g_mean = ND_FLOAT g_std = ND_FLOAT b_mean = ND_FLOAT b_std = ND_FLOAT gcc_mean = ND_FLOAT gcc_std = ND_FLOAT gcc_50 = ND_FLOAT gcc_75 = ND_FLOAT gcc_90 = ND_FLOAT rcc_mean = ND_FLOAT rcc_std = ND_FLOAT rcc_50 = ND_FLOAT rcc_75 = ND_FLOAT rcc_90 = ND_FLOAT max_solar_elev = max(solar_elev_vals) snow_flag = ND_INT outlierflag_gcc_mean = ND_INT outlierflag_gcc_50 = ND_INT outlierflag_gcc_75 = ND_INT outlierflag_gcc_90 = ND_INT # stats for this period should be complete - only # snow flags are missing data else: # find nearest image to midday (noon) on mid-interval date mi_ndx = midday_delta_vals.index(min(midday_delta_vals)) midday_filename = filenames[mi_ndx] midday_r = r_dn_vals[mi_ndx] midday_g = g_dn_vals[mi_ndx] midday_b = b_dn_vals[mi_ndx] midday_gcc = gcc_vals[mi_ndx] midday_rcc = rcc_vals[mi_ndx] # get stats for this time interval image_count = img_cnt r_mean = np.nanmean(r_dn_vals) r_std = np.nanstd(r_dn_vals) g_mean = np.nanmean(g_dn_vals) g_std = np.nanstd(g_dn_vals) b_mean = np.nanmean(b_dn_vals) b_std = np.nanstd(b_dn_vals) gcc_mean = np.nanmean(gcc_vals) gcc_std = np.nanstd(gcc_vals) gcc_50 = quantile(gcc_vals, 0.5) gcc_75 = quantile(gcc_vals, 0.75) gcc_90 = quantile(gcc_vals, 0.9) rcc_mean = np.nanmean(rcc_vals) rcc_std = np.nanstd(rcc_vals) rcc_50 = quantile(rcc_vals, 0.5) rcc_75 = quantile(rcc_vals, 0.75) rcc_90 = quantile(rcc_vals, 0.9) max_solar_elev = max(solar_elev_vals) snow_flag = ND_INT outlierflag_gcc_mean = ND_INT outlierflag_gcc_50 = ND_INT outlierflag_gcc_75 = ND_INT outlierflag_gcc_90 = ND_INT # append to gcc timeseries gcc_ts_row = gcc_ts.insert_row( gcc_date, doy, image_count, midday_filename, midday_r, midday_g, midday_b, midday_gcc, midday_rcc, r_mean, r_std, g_mean, g_std, b_mean, b_std, gcc_mean, gcc_std, gcc_50, gcc_75, gcc_90, rcc_mean, rcc_std, rcc_50, rcc_75, rcc_90, max_solar_elev, snow_flag, outlierflag_gcc_mean, outlierflag_gcc_50, outlierflag_gcc_75, outlierflag_gcc_90, ) # print(result if verbose) if verbose: csvstr = gcc_ts.format_csvrow(gcc_ts_row) print(csvstr) # reset accumulated values img_cnt = 0 filenames = [] r_dn_vals = [] rcc_vals = [] g_dn_vals = [] gcc_vals = [] b_dn_vals = [] bcc_vals = [] solar_elev_vals = [] midday_delta_vals = [] if dryrun: nout = 0 else: nout = gcc_ts.writeCSV(outpath) print("Total: %d" % (nout,))
def main(): # set up command line argument processing parser = argparse.ArgumentParser() # options parser.add_argument( "-v", "--verbose", help="increase output verbosity", action="store_true", default=False, ) parser.add_argument( "-n", "--dry-run", help="Process data but don't save results", action="store_true", default=False, ) # positional arguments parser.add_argument("site", help="PhenoCam site name") parser.add_argument("roiname", help="ROI name, e.g. canopy_0001") # get args args = parser.parse_args() sitename = args.site roiname = args.roiname verbose = args.verbose dryrun = args.dry_run if verbose: print("site: {0}".format(sitename)) print("roiname: {0}".format(roiname)) print("verbose: {0}".format(verbose)) print("dryrun: {0}".format(dryrun)) # set output filename inname = "%s_%s_roistats.csv" % (sitename, roiname) outname = inname inpath = os.path.join(archive_dir, sitename, "ROI", outname) outpath = inpath if verbose: print("output file: {0}".format(outname)) # get ROI list roi_list = get_roi_list(sitename, roiname) # read existing CSV file - since this is an update throw # exception if the file doesn't already exist try: roits = ROITimeSeries(site=sitename, ROIListID=roiname) roits.readCSV(outpath) except IOError: errmsg = "Unable to read CSV file: {0}\n".format(outpath) sys.stderr.write(errmsg) sys.exit(1) # read in config file for this site if it exists config_file = "{0}_{1}.cfg".format(sitename, roiname) config_path = os.path.join(archive_dir, sitename, "ROI", config_file) if os.path.exists(config_path): cfgparser = configparser(defaults={"resize": str(default_resize)}) cfgparser.read(config_path) if cfgparser.has_section("roi_timeseries"): resizeFlg = cfgparser.getboolean("roi_timeseries", "resize") else: resizeFlg = default_resize # verify that config matches CSV header! if resizeFlg != roits.resizeFlg: errmsg = "resize flag from config doesn't match CSV header\n" sys.stderr.write(errmsg) sys.exit(1) else: resizeFlg = default_resize # print config values if verbose: print("") print("ROI timeseries config:") print("======================") print("roi_list: ", "{0}_{1}_roi.csv".format(sitename, roiname)) if os.path.exists(config_path): print("config file: {0}".format(config_file)) else: print("config file: None") print("Resize Flag: ", resizeFlg) # get list of images already in CSV old_imglist = roits.get_image_list() # find last dt in current timeseries CSV nlast = len(roits.rows) - 1 dt_last = roits.rows[nlast]["datetime"] # add five seconds so that we don't reprocess last image dt_last = dt_last + timedelta(seconds=5) # start with images newer than last dt dt_start = dt_last if verbose: print("last image at: {0}".format(dt_last)) # loop over mask entries in ROI list nimage = 0 nupdate = 0 for imask, roimask in enumerate(roi_list.masks): roi_startDT = roimask["start_dt"] roi_endDT = roimask["end_dt"] # skip this ROI maskfile if it's validity interval ends # before last date before update if roi_endDT < dt_start: continue # start_date = roi_startDT.date() # end_date = roi_endDT.date() # start_time = roi_startDT.time() # end_time = roi_endDT.time() maskfile = roimask["maskfile"] # okay set the start datetime to the larger of dt_start (from # last row of existing timeseries CSV) and the beginning of # the ROI validity. We need to do this for the case where # there is a gap between last row of CSV and beginning of next # validity interval. This will often be the case when there # are a series of "transitional images" between two # stable/useful camera positions. if dt_start < roi_startDT: dt_start = roi_startDT mask_path = os.path.join(archive_dir, sitename, "ROI", maskfile) # print roi_path try: mask_img = Image.open(mask_path) except Exception: sys.stderr.write("Unable to open ROI mask file\n") sys.exit(1) # check that mask_img is in expected form mask_mode = mask_img.mode if mask_mode != "L": # convert to 8-bit mask mask_img = mask_img.convert("L") # make a numpy mask roimask = np.asarray(mask_img, dtype=np.bool8) # get list of images for this timeperiod imglist = utils.getsiteimglist( sitename, getIR=False, startDT=dt_start, endDT=roi_endDT ) nimage += len(imglist) for impath in imglist: if debug: print(maskfile, impath) # check if image already exists in list -- just to be # sure! fn = os.path.basename(impath) try: row_index = old_imglist.index(fn) except Exception: row_index = None # append/insert row for this image/mask - shouldn't happen # but just to be on safe side! if row_index: roits_row = roits.insert_row(impath, roimask, imask + 1) else: roits_row = roits.append_row(impath, roimask, imask + 1) # check that we could append/insert a row if roits_row: nupdate += 1 else: continue if verbose: csvstr = roits.format_csvrow(roits_row) print(csvstr) if debug: if nupdate == 10: break # output CSV file if dryrun: nout = 0 else: nout = roits.writeCSV(outpath) print("Images processed: %d" % (nimage,)) print("Images added to CSV: %d" % (nupdate,)) print("Total: %d" % (nout,))
def main(): """ generate IR ROI timeseries from a PhenoCam directory of images """ # set up command line argument processing parser = argparse.ArgumentParser() # options parser.add_argument( "-v", "--verbose", help="increase output verbosity", action="store_true", default=False, ) parser.add_argument( "-n", "--dry-run", help="Process data but don't save results", action="store_true", default=False, ) # positional arguments parser.add_argument("site", help="PhenoCam site name") parser.add_argument("roiname", help="ROI name, e.g. DB_0001") # get args args = parser.parse_args() sitename = args.site roiname = args.roiname verbose = args.verbose dryrun = args.dry_run if verbose: print("site: {0}".format(sitename)) print("roiname: {0}".format(roiname)) print("verbose: {0}".format(verbose)) print("dryrun: {0}".format(dryrun)) # set output filename outname = "%s_%s_IR_roistats.csv" % (sitename, roiname) outdir = os.path.join(archive_dir, sitename, "ROI") outpath = os.path.join(outdir, outname) if verbose: print("archive dir: {0}".format(archive_dir)) print("output file: {0}".format(outname)) # read in config file for this site if it exists config_file = "{0}_{1}.cfg".format(sitename, roiname) config_path = os.path.join(archive_dir, sitename, "ROI", config_file) if os.path.exists(config_path): cfgparser = configparser(defaults={"resize": str(default_resize)}) cfgparser.read(config_path) if cfgparser.has_section("roi_timeseries"): resizeFlg = cfgparser.getboolean("roi_timeseries", "resize") else: resizeFlg = default_resize else: resizeFlg = default_resize # print config values if verbose: print("") print("ROI IR timeseries config:") print("=========================") print("roi_list: ", "{0}_{1}_roi.csv".format(sitename, roiname)) if os.path.exists(config_path): print("config file: {0}".format(config_file)) else: print("config file: None") print("Resize Flag: ", resizeFlg) # create new roi_timeseries object for this ROIList roits = IRROITimeSeries(site=sitename, ROIListID=roiname, resizeFlag=resizeFlg) # grab roi list roi_list = get_roi_list(sitename, roiname) # loop over mask entries in ROI list nimage = 0 nupdate = 0 for roimask_index, roimask in enumerate(roi_list.masks): startDT = roimask["start_dt"] endDT = roimask["end_dt"] maskfile = roimask["maskfile"] mask_path = os.path.join(archive_dir, sitename, "ROI", maskfile) # open roi mask file try: mask_img = Image.open(mask_path) except IOError: sys.stderr.write("Unable to open ROI mask file\n") sys.exit(1) # check that mask_img is in expected form mask_mode = mask_img.mode if mask_mode != "L": # convert to 8-bit mask mask_img = mask_img.convert("L") # make a numpy mask roimask = np.asarray(mask_img, dtype=np.bool8) # get list of images for this timeperiod imglist = utils.getsiteimglist(sitename, getIR=True, startDT=startDT, endDT=endDT) nimage += len(imglist) for impath in imglist: # append row for this image/mask - shouldn't get # any duplicates so just append roits_row = roits.append_row(impath, roimask, roimask_index + 1) if roits_row: nupdate += 1 else: continue if verbose: csvstr = roits.format_csvrow(roits_row) print(csvstr) if debug: if nupdate == 10: break # output CSV file if dryrun: nout = 0 else: nout = roits.writeCSV(outpath) print("Images processed: %d" % (nimage, )) print("Images added to CSV: %d" % (nupdate, )) print("Total: %d" % (nout, ))
# # Copyright Leonardo Amaral <*****@*****.**> # # Inspired in http://lkml.indiana.edu/hypermail/linux/kernel/0811.3/01905.html # # Thanks To: Ricardo Canale <*****@*****.**> # And his mother for the Notebook Donation ;) # from configparser import ConfigParser as configparser from time import sleep from syslog import syslog, openlog, LOG_DAEMON, LOG_INFO, LOG_WARNING, LOG_NOTICE from os import getpid import struct, signal, sys opts = configparser() opts.read('/etc/driveguardd.conf') PID = getpid() openlog('Driveguardd [%d]' % PID, 0, LOG_DAEMON) with open('/var/run/driveguardd.pid', 'w') as pidfile: pidfile.write(str(PID) + '\n') def exit_handler(signalsent, frame): if signalsent == signal.SIGTERM or signalsent == signal.SIGINT: syslog(LOG_NOTICE, 'Stopping daemon.') sys.exit(0) else:
argument_parser.add_argument( "-d", "--desktop", type=str, help="Switch to desktop DESKTOP, as defined in the config.", required=True) argument_parser.add_argument("--config", type=str, default=expanduser("~") + "/.config/switcher.conf", help="Set the config file location.") # Parse the arguments. arguments = argument_parser.parse_args() # Create a parser for the config file. config = configparser() # Chech that the config file exists and read it. if isfile(arguments.config): config.read(arguments.config) else: argument_parser.print_usage() communicate("error: Config file '%s' not found." % (arguments.config), quit=True) # Check the arguments and the config. if arguments.desktop not in config.sections(): argument_parser.print_usage() communicate( "error: Unsupported desktop given. Supported values are: '%s'." % ("', '".join(config.sections())), quit=True)
import logging.config import logging import yaml import configparser from sqlalchemy.orm import sessionmaker from sqlalchemy import create_engine from collector.pastebin.paste_collector import PasteParser from models.model import PasteData, HitData if __name__ == '__main__': with open('/users/paco/proyectos/osint/core/collector/log.yaml', 'r') as f: config = yaml.safe_load(f.read()) logging.config.dictconfig(config) logger = logging.getlogger('pastes') cfg = configparser.configparser() cfg.read('/users/paco/proyectos/osint/core/collector/config.ini') engine = create_engine('mysql+pymysql://dev:xxxxx@localhost/testing') session = sessionmaker(bind=engine) session = session() data_sources = {'paste': pastedata(session), 'hit': hitdata(session)} pasteparser(data_sources['paste'], data_sources['hit'], cfg['pastebin']).start_scrapping()
#Script to check the number of unreviewed tweets and email the admins if there are some import MySQLdb import tweetsclient import smtplib from email.mime.text import MIMEText import configparser import pytz import datetime smtpconfig = configparser.configparser() smtpconfig.read('conf/tweets-client.ini') smtp = smtplib.SMTP(smtpconfig.get('moderation-alerts', 'mail_host'), smtpconfig.get('moderation-alerts', 'mail_port')) smtp.login(smtpconfig.get('moderation-alerts', 'mail_username'), smtpconfig.get('moderation-alerts', 'mail_password')) recipient = smtpconfig.get('moderation-alerts', 'unmoderated_recipient') sender = smtpconfig.get('moderation-alerts', 'sender') max_tweets = smtpconfig.getint('moderation-alerts', 'max_tweets') config = tweetsclient.Config().get() conn = MySQLdb.connect( host=config.get('database', 'host'), port=int(config.get('database', 'port')), db=config.get('database', 'database'), user=config.get('database', 'username'), passwd=config.get('database', 'password'), charset="utf8mb4", use_unicode=True ) cur = conn.cursor()
_image.set_colorkey(colorkey) _image = _image.convert() else: _image = _image.convert_alpha() return _image # import globals import configparser, os # Resource loading: DATA_PY = os.path.abspath(os.path.dirname(__file__)) DATA_DIR = os.path.normpath(os.path.join(DATA_PY, '..', 'data/config/')) """ Parse Globals File """ global_config = configparser.configparser() global_config.read(os.path.join(DATA_DIR, "Globals.ini")) G, D = {}, {} for pair in global_config.items('game'): if pair[1].isdigit() == True: value = int(pair[1]) elif pair[1].lower() == "true": value = True elif pair[1].lower() == "false": value = False else: value = pair[1] G[pair[0]] = value