def test_get_config_value_dtype_None(self): config = RawConfigParser() config["section"] = {"key": "val"} val = get_config_value(config, "section", "key") self.assertEqual(val, "val")
""" Django settings for texada project. """ import os import logging from configparser import RawConfigParser config = RawConfigParser() config.read('/etc/texada_settings/settings.ini') BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) ROOT_URLCONF = 'texadadjango.urls' # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = config.get('secrets', 'SECRET_KEY') #debugging DEBUG_API = config.getboolean('django', 'DEBUG_API') # SECURITY WARNING: don't run with debug turned on in production! DEBUG = config.getboolean('django', 'DEBUG') ALLOWED_HOSTS = ['192.168.56.101', '127.0.0.1', 'localhost'] # Application definition INSTALLED_APPS = [ 'django.contrib.admin', 'django.contrib.auth', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', 'rest_framework',
def entry(): _deprecation_check(sys.argv[0]) from django.core.exceptions import ImproperlyConfigured from django.core.management import execute_from_command_line, find_commands from django.core.management import CommandParser from django.core.management.base import BaseCommand os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'desktop.settings') cm_config_file = '/etc/cloudera-scm-agent/config.ini' ld_path_orig = None if "LD_LIBRARY_PATH" in list(os.environ.keys()): ld_path_orig = os.environ["LD_LIBRARY_PATH"] # What's the subcommand being run? # This code uses the same logic from django.core.management to handle command args subcommand = None if "--skip-reload" in sys.argv: skip_reload = True sys.argv.remove("--skip-reload") else: skip_reload = False # Check if --cm-managed flag is set and strip it out # to prevent from sending to subcommands if "--cm-managed" in sys.argv: sys.argv.remove("--cm-managed") cm_managed = True else: cm_managed = False if len(sys.argv) > 1: subcommand = sys.argv[1] parser = CommandParser(None, usage="%(prog)s subcommand [options] [args]", add_help=False) parser.parse_known_args(sys.argv[2:]) if len(sys.argv) > 1: prof_id = subcommand = sys.argv[1] #Check if this is a CM managed cluster if os.path.isfile( cm_config_file) and not cm_managed and not skip_reload: print("ALERT: This appears to be a CM Managed environment") print( "ALERT: HUE_CONF_DIR must be set when running hue commands in CM Managed environment" ) print("ALERT: Please run 'hue <command> --cm-managed'") else: prof_id = str(os.getpid()) # CM managed configure env vars if cm_managed: if sys.version_info[0] > 2: from configparser import NoOptionError, RawConfigParser else: from ConfigParser import NoOptionError, RawConfigParser config = RawConfigParser() config.read(cm_config_file) try: cm_agent_run_dir = config.get( 'General', 'agent_wide_credential_cache_location') except NoOptionError: cm_agent_run_dir = '/var/run/cloudera-scm-agent' pass #Parse CM supervisor include file for Hue and set env vars cm_supervisor_dir = cm_agent_run_dir + '/supervisor/include' cm_process_dir = cm_agent_run_dir + '/process' hue_env_conf = None envline = None cm_hue_string = "HUE_SERVER" for file in os.listdir(cm_supervisor_dir): if cm_hue_string in file: hue_env_conf = file hue_env_conf = cm_supervisor_dir + "/" + hue_env_conf if hue_env_conf == None: process_dirs = fnmatch.filter(os.listdir(cm_process_dir), '*%s*' % cm_hue_string) process_dirs.sort() hue_process_dir = cm_process_dir + "/" + process_dirs[-1] hue_env_conf = fnmatch.filter(os.listdir(hue_process_dir), 'supervisor.conf')[0] hue_env_conf = hue_process_dir + "/" + hue_env_conf if not hue_env_conf == None: if os.path.isfile(hue_env_conf): hue_env_conf_file = open(hue_env_conf, "r") for line in hue_env_conf_file: if "environment" in line: envline = line if "directory" in line: empty, hue_conf_dir = line.split("directory=") os.environ["HUE_CONF_DIR"] = hue_conf_dir.rstrip() else: print("This appears to be a CM managed cluster, but the") print("supervisor/include file for Hue could not be found") print("in order to successfully run commands that access") print("the database you need to set the following env vars:") print("") print(" export JAVA_HOME=<java_home>") print( " export HUE_CONF_DIR=\"%s/`ls -1 %s | grep %s | sort -n | tail -1 `\"" % (cm_processs_dir, cm_process_dir, cm_hue_string)) print(" export HUE_IGNORE_PASSWORD_SCRIPT_ERRORS=1") print(" export HUE_DATABASE_PASSWORD=<hueDBpassword>") print("If using Oracle as your database:") print(" export LD_LIBRARY_PATH=/path/to/instantclient") print("") print( "If the above does not work, make sure Hue has been started on this server." ) if not envline == None: empty, environment = envline.split("environment=") for envvar in environment.split(","): include_env_vars = ("HADOOP_C", "PARCEL", "SCM_DEFINES", "LD_LIBRARY") if any(include_env_var in envvar for include_env_var in include_env_vars): envkey, envval = envvar.split("=") envval = envval.replace("'", "").rstrip() os.environ[envkey] = envval #Set JAVA_HOME if "JAVA_HOME" not in list(os.environ.keys()): if os.path.isfile( '/usr/lib64/cmf/service/common/cloudera-config.sh'): locate_java = subprocess.Popen([ 'bash', '-c', '. /usr/lib64/cmf/service/common/cloudera-config.sh; locate_java_home' ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) elif os.path.isfile( '/opt/cloudera/cm-agent/service/common/cloudera-config.sh' ): locate_java = subprocess.Popen([ 'bash', '-c', '. /opt/cloudera/cm-agent/service/common/cloudera-config.sh; locate_java_home' ], stdout=subprocess.PIPE, stderr=subprocess.PIPE) else: locate_java = None JAVA_HOME = "UNKNOWN" if locate_java is not None: for line in iter(locate_java.stdout.readline, ''): if 'JAVA_HOME' in line: JAVA_HOME = line.rstrip().split('=')[1] if JAVA_HOME != "UNKNOWN": os.environ["JAVA_HOME"] = JAVA_HOME if "JAVA_HOME" not in list(os.environ.keys()): print( "JAVA_HOME must be set and can't be found, please set JAVA_HOME environment variable" ) print(" export JAVA_HOME=<java_home>") sys.exit(1) #Make sure we set Oracle Client if configured if "LD_LIBRARY_PATH" not in list(os.environ.keys()): if "SCM_DEFINES_SCRIPTS" in list(os.environ.keys()): for scm_script in os.environ["SCM_DEFINES_SCRIPTS"].split(":"): if "ORACLE" in scm_script: if os.path.isfile(scm_script): oracle_source = subprocess.Popen( ". %s; env" % scm_script, stdout=subprocess.PIPE, shell=True, executable="/bin/bash") for line in oracle_source.communicate( )[0].splitlines(): if "LD_LIBRARY_PATH" in line: var, oracle_ld_path = line.split("=") os.environ[ "LD_LIBRARY_PATH"] = oracle_ld_path if "LD_LIBRARY_PATH" not in list(os.environ.keys()): print( "LD_LIBRARY_PATH can't be found, if you are using ORACLE for your Hue database" ) print("then it must be set, if not, you can ignore") print(" export LD_LIBRARY_PATH=/path/to/instantclient") if "LD_LIBRARY_PATH" in list(os.environ.keys()): if ld_path_orig is not None and ld_path_orig == os.environ[ "LD_LIBRARY_PATH"]: skip_reload = True if not skip_reload: reload_with_cm_env(cm_managed) try: # Let django handle the normal execution if os.getenv("DESKTOP_PROFILE"): _profile(prof_id, lambda: execute_from_command_line(sys.argv)) else: execute_from_command_line(sys.argv) except ImproperlyConfigured as e: if len(sys.argv) > 1 and sys.argv[ 1] == 'is_db_alive' and 'oracle' in str(e).lower(): print(e, file=sys.stderr) # Oracle connector is improperly configured sys.exit(10) else: raise e except subprocess.CalledProcessError as e: if "altscript.sh" in str(e).lower(): print("%s" % e) print( "HUE_CONF_DIR seems to be set to CM location and '--cm-managed' flag not used" )
def get_merged_config(self, **options): """Get the final merged configuration for supvervisord, as a string. This is the top-level function exported by this module. It combines the config file from the main project with default settings and those specified in the command-line, processes various special section names, and returns the resulting configuration as a string. """ config_file = "supervisord.conf" # Initialise the ConfigParser. # Fortunately for us, ConfigParser has merge-multiple-config-files # functionality built into it. You just read each file in turn, and # values from later files overwrite values from former. cfg = RawConfigParser() # Start from the default configuration options. cfg.readfp(StringIO(self.DEFAULT_CONFIG)) """ # Add in the project-specific config file. with open(config_file,"r") as f: data = f.read() cfg.readfp(StringIO(data)) """ # Add in the options from the self.services cfg.readfp(StringIO(self.get_config_from_services())) # Add in the options specified on the command-line. cfg.readfp(StringIO(self.get_config_from_options(**options))) # Add options from [program:__defaults__] to each program section # if it happens to be missing that option. PROG_DEFAULTS = "program:__defaults__" if cfg.has_section(PROG_DEFAULTS): for option in cfg.options(PROG_DEFAULTS): default = cfg.get(PROG_DEFAULTS,option) for section in cfg.sections(): if section.startswith("program:"): if not cfg.has_option(section,option): cfg.set(section,option,default) cfg.remove_section(PROG_DEFAULTS) # Add options from [program:__overrides__] to each program section # regardless of whether they already have that option. PROG_OVERRIDES = "program:__overrides__" if cfg.has_section(PROG_OVERRIDES): for option in cfg.options(PROG_OVERRIDES): override = cfg.get(PROG_OVERRIDES,option) for section in cfg.sections(): if section.startswith("program:"): cfg.set(section,option,override) cfg.remove_section(PROG_OVERRIDES) # Make sure we've got a port configured for supervisorctl to # talk to supervisord. It's passworded based on secret key. # If they have configured a unix socket then use that, otherwise # use an inet server on localhost at fixed-but-randomish port. username = hashlib.md5("angelo".encode('utf-8')).hexdigest()[:7] password = hashlib.md5(username.encode('utf-8')).hexdigest() if cfg.has_section("unix_http_server"): self.set_if_missing(cfg,"unix_http_server","username",username) self.set_if_missing(cfg,"unix_http_server","password",password) serverurl = "unix://" + cfg.get("unix_http_server","file") else: # This picks a "random" port in the 9000 range to listen on. # It's derived from the secret key, so it's stable for a given # project but multiple projects are unlikely to collide. port = int(hashlib.md5(password.encode('utf-8')).hexdigest()[:3],16) % 1000 addr = "127.0.0.1:9%03d" % (port,) self.set_if_missing(cfg,"inet_http_server","port",addr) self.set_if_missing(cfg,"inet_http_server","username",username) self.set_if_missing(cfg,"inet_http_server","password",password) serverurl = "http://" + cfg.get("inet_http_server","port") self.set_if_missing(cfg,"supervisorctl","serverurl",serverurl) self.set_if_missing(cfg,"supervisorctl","username",username) self.set_if_missing(cfg,"supervisorctl","password",password) self.set_if_missing(cfg,"rpcinterface:supervisor", "supervisor.rpcinterface_factory", "supervisor.rpcinterface:make_main_rpcinterface") # Remove any [program:] sections with exclude=true for section in cfg.sections(): try: if cfg.getboolean(section,"exclude"): cfg.remove_section(section) except NoOptionError: pass # Sanity-check to give better error messages. for section in cfg.sections(): if section.startswith("program:"): if not cfg.has_option(section,"command"): msg = "Process name '%s' has no command configured" raise ValueError(msg % (section.split(":",1)[-1])) # Write it out to a StringIO and return the data s = StringIO() cfg.write(s) return s.getvalue()
def read_file(self): here = "config.read_file" config_read = RawConfigParser() config_read.read(self.config_filename) section = "Debug" self.debug_reread_config = str2bool( config_read.get(section, 'debug_reread_config')) self.debug_flag_1 = str2bool(config_read.get(section, 'debug_flag_1')) self.debug_flag_2 = str2bool(config_read.get(section, 'debug_flag_2')) self.debug_flag_ftp = str2bool( config_read.get(section, 'debug_flag_ftp')) section = "Scan" self.scan_delay = float(config_read.get(section, 'scan_delay')) self.max_scans = float(config_read.get(section, 'max_scans')) section = "Log" self.log_directory = config_read.get(section, 'log_directory') self.local_dir_www = config_read.get(section, 'local_dir_www') self.log_buffer_flag = config_read.getboolean(section, 'log_buffer_flag') self.text_buffer_length = int( config_read.get(section, 'text_buffer_length')) section = "Ftp" self.ftp_creds_filename = config_read.get(section, 'ftp_creds_filename') self.ftp_log_max_count = float( config_read.get(section, 'ftp_log_max_count')) section = "Heating_Fan" self.heat_max_temp = float(config_read.get(section, 'heat_max_temp')) self.heat_min_temp = float(config_read.get(section, 'heat_min_temp')) self.heat_max_speed = float(config_read.get(section, 'heat_max_speed')) self.heat_min_speed = float(config_read.get(section, 'heat_min_speed')) self.heat_max_freq = float(config_read.get(section, 'heat_max_freq')) self.heat_min_freq = float(config_read.get(section, 'heat_min_freq')) section = "Sauna" self.sauna_max_temp = float(config_read.get(section, 'sauna_max_temp')) self.sauna_min_temp = float(config_read.get(section, 'sauna_min_temp')) self.sauna_max_speed = float( config_read.get(section, 'sauna_max_speed')) self.sauna_min_speed = float( config_read.get(section, 'sauna_min_speed')) self.sauna_max_freq = float(config_read.get(section, 'sauna_max_freq')) self.sauna_min_freq = float(config_read.get(section, 'sauna_min_freq')) self.sauna_GPIO_port = float( config_read.get(section, 'sauna_GPIO_port')) self.sauna_brightness = float( config_read.get(section, 'sauna_brightness')) section = "Power_Log" self.adc_scan_size = int(config_read.get(section, 'adc_scan_size')) self.adc_target_scan_msec = int( config_read.get(section, 'adc_target_scan_msec')) self.adc_channel = int(config_read.get(section, 'adc_channel')) self.adc_default_gain = int( config_read.get(section, 'adc_default_gain')) self.adc_top_limit = int(config_read.get(section, 'adc_top_limit')) self.adc_bottom_limit = int( config_read.get(section, 'adc_bottom_limit')) self.adc_input_offset_mv = float( config_read.get(section, 'adc_input_offset_mv')) self.adc_input_amp_gain = float( config_read.get(section, 'adc_input_amp_gain')) self.adc_CT_ratio = float(config_read.get(section, 'adc_CT_ratio')) self.adc_CT_resister = float( config_read.get(section, 'adc_CT_resister')) return
def ini_to_dict(fname, section): """Convert *section* of .ini *config* to dictionary.""" from configparser import RawConfigParser, NoOptionError config = RawConfigParser() config.read(fname) conf = {} conf['posttroll'] = {} posttroll = conf['posttroll'] posttroll['topics'] = config.get(section, 'topics').split() try: nameservers = config.get(section, 'nameservers') nameservers = nameservers.split() except (NoOptionError, ValueError): nameservers = None posttroll['nameservers'] = nameservers try: addresses = config.get(section, 'addresses') addresses = addresses.split() except (NoOptionError, ValueError): addresses = None posttroll['addresses'] = addresses try: services = config.get(section, 'services') services = services.split() except (NoOptionError, ValueError): services = "" posttroll['services'] = services try: publish_port = config.get(section, 'publish_port') except NoOptionError: publish_port = 0 posttroll['publish_port'] = publish_port posttroll['publish_topic'] = config.get(section, "publish_topic") conf['patterns'] = {section: {}} patterns = conf['patterns'][section] patterns['pattern'] = config.get(section, 'pattern') patterns['critical_files'] = config.get(section, 'critical_files') patterns['wanted_files'] = config.get(section, 'wanted_files') patterns['all_files'] = config.get(section, 'all_files') patterns['is_critical_set'] = False try: patterns['variable_tags'] = config.get(section, 'variable_tags').split(',') except NoOptionError: patterns['variable_tags'] = [] try: conf['time_tolerance'] = config.getint(section, "time_tolerance") except NoOptionError: conf['time_tolerance'] = 30 try: # Seconds conf['timeliness'] = config.getint(section, "timeliness") except (NoOptionError, ValueError): conf['timeliness'] = 1200 try: conf['num_files_premature_publish'] = \ config.getint(section, "num_files_premature_publish") except (NoOptionError, ValueError): conf['num_files_premature_publish'] = -1 try: conf['group_by_minutes'] = config.getint(section, 'group_by_minutes') except (NoOptionError, ValueError): pass try: kps = config.get(section, 'keep_parsed_keys') conf['keep_parsed_keys'] = kps.split() except NoOptionError: pass try: conf['providing_server'] = config.get(section, "providing_server") except (NoOptionError, ValueError): conf['providing_server'] = None try: conf['time_name'] = config.get(section, "time_name") except (NoOptionError, ValueError): conf['time_name'] = 'start_time' try: conf['check_existing_files_after_start'] = config.getboolean( section, "check_existing_files_after_start") except (NoOptionError, ValueError): conf['check_existing_files_after_start'] = False return conf
def export_eds(od, dest=None, file_info={}, device_commisioning=False): def export_object(obj, eds): if type(obj) is objectdictionary.Variable: return export_variable(obj, eds) if type(obj) is objectdictionary.Record: return export_record(obj, eds) if type(obj) is objectdictionary.Array: return export_array(obj, eds) def export_common(var, eds, section): eds.add_section(section) eds.set(section, "ParameterName", var.name) if var.storage_location: eds.set(section, "StorageLocation", var.storage_location) def export_variable(var, eds): if type(var.parent) is objectdictionary.ObjectDictionary: # top level variable section = "%04X" % var.index else: # nested variable section = "%04Xsub%X" % (var.index, var.subindex) export_common(var, eds, section) eds.set(section, "ObjectType", "0x%X" % VAR) if var.data_type: eds.set(section, "DataType", "0x%04X" % var.data_type) if var.access_type: eds.set(section, "AccessType", var.access_type) if getattr(var, 'default_raw', None) is not None: eds.set(section, "DefaultValue", var.default_raw) elif getattr(var, 'default', None) is not None: eds.set(section, "DefaultValue", _revert_variable(var.data_type, var.default)) if device_commisioning: if getattr(var, 'value_raw', None) is not None: eds.set(section, "ParameterValue", var.value_raw) elif getattr(var, 'value', None) is not None: eds.set(section, "ParameterValue", _revert_variable(var.data_type, var.default)) eds.set(section, "DataType", "0x%04X" % var.data_type) eds.set(section, "PDOMapping", hex(var.pdo_mappable)) if getattr(var, 'min', None) is not None: eds.set(section, "LowLimit", var.min) if getattr(var, 'max', None) is not None: eds.set(section, "HighLimit", var.max) def export_record(var, eds): section = "%04X" % var.index export_common(var, eds, section) eds.set(section, "SubNumber", "0x%X" % len(var.subindices)) ot = RECORD if type(var) is objectdictionary.Record else ARR eds.set(section, "ObjectType", "0x%X" % ot) for i in var: export_variable(var[i], eds) export_array = export_record eds = RawConfigParser() # both disables lowercasing, and allows int keys eds.optionxform = str from datetime import datetime as dt defmtime = dt.utcnow() try: # only if eds was loaded by us origFileInfo = od.__edsFileInfo except AttributeError: origFileInfo = { # just set some defaults "CreationDate": defmtime.strftime("%m-%d-%Y"), "CreationTime": defmtime.strftime("%I:%m%p"), "EdsVersion": 4.2, } file_info.setdefault("ModificationDate", defmtime.strftime("%m-%d-%Y")) file_info.setdefault("ModificationTime", defmtime.strftime("%I:%m%p")) for k, v in origFileInfo.items(): file_info.setdefault(k, v) eds.add_section("FileInfo") for k, v in file_info.items(): eds.set("FileInfo", k, v) eds.add_section("DeviceInfo") for eprop, odprop in [ ("VendorName", "vendor_name"), ("VendorNumber", "vendor_number"), ("ProductName", "product_name"), ("ProductNumber", "product_number"), ("RevisionNumber", "revision_number"), ("OrderCode", "order_code"), ("SimpleBootUpMaster", "simple_boot_up_master"), ("SimpleBootUpSlave", "simple_boot_up_slave"), ("Granularity", "granularity"), ("DynamicChannelsSupported", "dynamic_channels_supported"), ("GroupMessaging", "group_messaging"), ("NrOfRXPDO", "nr_of_RXPDO"), ("NrOfTXPDO", "nr_of_TXPDO"), ("LSS_Supported", "LSS_supported"), ]: val = getattr(od.device_information, odprop, None) if type(val) is None: continue elif type(val) is str: eds.set("DeviceInfo", eprop, val) elif type(val) in (int, bool): eds.set("DeviceInfo", eprop, int(val)) # we are also adding out of spec baudrates here. for rate in od.device_information.allowed_baudrates.union( {10e3, 20e3, 50e3, 125e3, 250e3, 500e3, 800e3, 1000e3}): eds.set("DeviceInfo", "BaudRate_%i" % (rate / 1000), int(rate in od.device_information.allowed_baudrates)) if device_commisioning and (od.bitrate or od.node_id): eds.add_section("DeviceComissioning") if od.bitrate: eds.set("DeviceComissioning", "BaudRate", int(od.bitrate / 1000)) if od.node_id: eds.set("DeviceComissioning", "NodeID", int(od.node_id)) eds.add_section("Comments") i = 0 for line in od.comments.splitlines(): i += 1 eds.set("Comments", "Line%i" % i, line) eds.set("Comments", "Lines", i) eds.add_section("DummyUsage") for i in range(1, 8): key = "Dummy%04d" % i eds.set("DummyUsage", key, 1 if (key in od) else 0) def mandatory_indices(x): return x in {0x1000, 0x1001, 0x1018} def manufacturer_idices(x): return x in range(0x2000, 0x6000) def optional_indices(x): return all(( x > 0x1001, not mandatory_indices(x), not manufacturer_idices(x), )) supported_mantatory_indices = list(filter(mandatory_indices, od)) supported_optional_indices = list(filter(optional_indices, od)) supported_manufacturer_indices = list(filter(manufacturer_idices, od)) def add_list(section, list): eds.add_section(section) eds.set(section, "SupportedObjects", len(list)) for i in range(0, len(list)): eds.set(section, (i + 1), "0x%04X" % list[i]) for index in list: export_object(od[index], eds) add_list("MandatoryObjects", supported_mantatory_indices) add_list("OptionalObjects", supported_optional_indices) add_list("ManufacturerObjects", supported_manufacturer_indices) if not dest: import sys dest = sys.stdout eds.write(dest, False)
def parse_setup_cfg(): cfg = RawConfigParser() r = cfg.read([os.path.join(ROOTDIR, "setup.cfg")]) if len(r) != 1: print("Cannot read 'setup.cfg'") sys.exit(1) metadata = { "name": cfg.get("x-metadata", "name"), "version": cfg.get("x-metadata", "version"), "description": cfg.get("x-metadata", "description"), } _opt_value(cfg, metadata, "x-metadata", "license") _opt_value(cfg, metadata, "x-metadata", "maintainer") _opt_value(cfg, metadata, "x-metadata", "maintainer_email") _opt_value(cfg, metadata, "x-metadata", "author") _opt_value(cfg, metadata, "x-metadata", "author_email") _opt_value(cfg, metadata, "x-metadata", "url") _opt_value(cfg, metadata, "x-metadata", "download_url") _opt_value(cfg, metadata, "x-metadata", "classifiers", _as_lines) _opt_value(cfg, metadata, "x-metadata", "platforms", _as_list) _opt_value(cfg, metadata, "x-metadata", "packages", _as_list) _opt_value(cfg, metadata, "x-metadata", "keywords", _as_list) try: v = cfg.get("x-metadata", "requires-dist") except (NoOptionError, NoSectionError): pass else: requires = _as_requires(v) if requires: metadata["install_requires"] = requires try: v = cfg.get("x-metadata", "requires-test") except (NoOptionError, NoSectionError): pass else: requires = _as_requires(v) if requires: metadata["tests_require"] = requires try: v = cfg.get("x-metadata", "long_description_file") except (NoOptionError, NoSectionError): pass else: parts = [] for nm in v.split(): fp = open(nm, "r") parts.append(fp.read()) fp.close() metadata["long_description"] = "\n\n".join(parts) try: v = cfg.get("x-metadata", "zip-safe") except (NoOptionError, NoSectionError): pass else: metadata["zip_safe"] = _as_bool(v) try: v = cfg.get("x-metadata", "console_scripts") except (NoOptionError, NoSectionError): pass else: if "entry_points" not in metadata: metadata["entry_points"] = {} metadata["entry_points"]["console_scripts"] = v.splitlines() if sys.version_info[:2] <= (2, 6): try: metadata["tests_require"] += ", unittest2" except KeyError: metadata["tests_require"] = "unittest2" if cfg.has_option("x-metadata", "ext_modules"): extensions = [] for ext in _as_list(cfg.get("x-metadata", "ext_modules")): ext = _extract_extension(cfg, ext) if ext is not None: extensions.append(ext) if extensions: metadata["ext_modules"] = [ Extension(*args, **kwds) for (args, kwds) in extensions ] return metadata
class GlobalVars: false_positives = [] whitelisted_users = [] blacklisted_users = [] blacklisted_usernames = [] blacklisted_websites = [] bad_keywords = [] watched_keywords = {} ignored_posts = [] auto_ignored_posts = [] startup_utc = datetime.utcnow().strftime("%H:%M:%S") latest_questions = [] api_backoff_time = 0 charcoal_room_id = "11540" meta_tavern_room_id = "89" socvr_room_id = "41570" blockedTime = { "all": 0, charcoal_room_id: 0, meta_tavern_room_id: 0, socvr_room_id: 0 } metasmoke_last_ping_time = datetime.now() not_privileged_warning = """ You are not a privileged user. Please see [the privileges wiki page](https://charcoal-se.org/smokey/Privileges) for information on what privileges are and what is expected of privileged users. """.strip().replace("\n", " ") experimental_reasons = [ # Don't widely report these "potentially bad keyword in answer", "potentially bad keyword in body", "potentially bad keyword in title", "potentially bad keyword in username" ] non_socvr_reasons = [] # Don't report to SOCVR non_tavern_reasons = [ # Don't report in the Tavern "all-caps body", "all-caps answer", "repeating characters in body", "repeating characters in title", "repeating characters in answer", "few unique characters in body", "few unique characters in answer", "title has only one unique char", "phone number detected in title", "offensive body detected", "no whitespace in body", "no whitespace in answer", ] non_tavern_sites = ["stackoverflow.com"] parser = HTMLParser() parser.unescape = unescape wrap = Client("stackexchange.com") wrapm = Client("meta.stackexchange.com") wrapso = Client("stackoverflow.com") privileged_users = { charcoal_room_id: [ "117490", # Normal Human "66258", # Andy "31768", # ManishEarth "103081", # hichris123 "73046", # Undo "88521", # ProgramFOX "59776", # Doorknob "31465", # Seth "88577", # Santa Claus "34124", # Andrew Leach "54229", # apnorton "20459", # S.L. Barth "32436", # tchrist "30477", # Brock Adams "58529", # ferrybig "145208", # Robert Longson "178825", # Ms Yvette "171800", # JAL "64978", # PeterJ "125141", # Jeffrey Bosboom "54902", # bummi "135450", # M.A.R. "145604", # Quill "60548", # rene "121401", # michaelpri "116218", # JamesENL "82927", # Braiam "11606", # bwDraco "19761", # Ilmari Karonen "108271", # Andrew T. "171054", # Magisch "190011", # Petter Friberg "165661", # Tunaki "145086", # Wai Ha Lee "137665", # ByteCommander "147884", # wythagoras "186395", # Åna "181293", # Ashish Ahuja "163686", # Gothdo "145827", # angussidney "244748", # Supreme Leader SnokeDetector (angussidney's sock) "121520", # ArtOfCode "244382", # Lt. A. Code (ArtOfCode's sock to test things with) "137388", # QPaysTaxes "212311", # Ryan Bemrose "172397", # Kyll "224538", # FrankerZ "61202", # OldSkool "56166", # Jan Dvorak "133966", # DavidPostill "22839", # djsmiley2k "97389", # Kaz Wolfe "144962", # DJMcMayhem "139423", # NobodyNada "62118", # tripleee "130558", # Registered User "128113", # arda "164318", # Glorfindel "175347", # Floern "180274", # Alexander O'Mara "158742", # Rob "207356", # 4castle "133031", # Mithrandir "215671", # Locutus of Borg (Mithrandir's Sock) "169713", # Mego "126657", # Cerbrus "10145", # Thomas Ward "161943", # J F "195967", # CaffeineAddiction "5363", # Stijn "248139", # FelixSFD "156721", # D-side "167070", # quartata "172450", # Hovercraft Full Of Eels "56200", # Eric Leschinski "211021", # Henders "255290", # Gypsy Spellweaver "64521", # CalvT "165474", # Hyper Neutrino "281362", # Hyper Neutrino v2 "169252", # Cai "155243", # Nisse Engström "69330", # Sconibulus "164187", # Okx "202619", # John Militer "262693", # suraj "11287", # Martin Sleziak "88588", # NVZ "281674", # paper1111 "279119", # Tetsuya Yamamoto "307652", # Ajay Brahmakshatriya "238145", # Owen Hines "268731", # Sagar V "205208", # a-j "234375", # Jarko Dubbeldam "139041", # Catija "305737", # Tinkeringbell "210948" # WELZ ], meta_tavern_room_id: [ "315433", # Normal Human "244519", # CRABOLO "244382", # TGMCians "194047", # Jan Dvorak "158100", # rene "178438", # Manishearth "237685", # hichris123 "215468", # Undo "229438", # ProgramFOX "180276", # Doorknob "161974", # Lynn Crumbling "186281", # Andy "266094", # Unihedro "245167", # Infinite Recursion "230261", # Jason C "213575", # Braiam "241919", # Andrew T. "203389", # backwards-Seth "202832", # Mooseman "160017", # bwDraco "201151", # bummi "188558", # Frank "229166", # Santa Claus "159034", # Kevin Brown "203972", # PeterJ "188673", # Alexis King "258672", # AstroCB "227577", # Sam "255735", # cybermonkey "279182", # Ixrec "271104", # James "220428", # Qantas 94 Heavy "153355", # tchrist "238426", # Ed Cottrell "166899", # Second Rikudo "287999", # ASCIIThenANSI "208518", # JNat "284141", # michaelpri "260312", # vaultah "244062", # SouravGhosh "152859", # Shadow Wizard "201314", # apnorton "280934", # M.A.Ramezani "200235", # durron597 "148310", # Awesome Poodles / Brock Adams "168333", # S.L. Barth "257207", # Unikitty "244282", # DroidDev "163250", # Cupcake "298265", # BoomsPlus "253560", # josilber "244254", # misterManSam "188189", # Robert Longson "174699", # Ilmari Karonen "202362", # chmod 666 telkitty "289717", # Quill "237813", # bjb568 "311345", # Simon Klaver "171881", # rekire "260388", # Pandya "310756", # Ms Yvette "262399", # Jeffrey Bosboom "242209", # JAL "280883", # ByteCommander "302251", # kos "262823", # ArtOfCode "215067", # Ferrybig "308386", # Magisch "285368", # angussidney "158829", # Thomas Ward "294691", # Mithrandir "203553", # CalvT "289971", # Hyper Neutrino "346854", # DonQuiKong "284336" # Catija ], socvr_room_id: [ "1849664", # Undo "2581872", # hichris123 "1198729", # Manishearth "3717023", # Normal Human aka 1999 "2619912", # ProgramFOX "578411", # rene "1043380", # gunr2171 "2246344", # Sam "2756409", # TylerH "1768232", # durron597 "359284", # Kevin Brown "258400", # easwee "3622940", # Unihedron "3204551", # Deduplicator "4342498", # NathanOliver "4639281", # Tiny Giant "3093387", # josilber "1652962", # cimmanon "1677912", # Mogsdad "656243", # Lynn Crumbling "3933332", # Rizier123 "2422013", # cybermonkey "3478852", # Nisse Engström "2302862", # Siguza "1324", # Paul Roub "1743880", # Tunaki "1663001", # DavidG "2415822", # JAL "4174897", # Kyll "5299236", # Kevin Guan "4050842", # Thaillie "1816093", # Drew "874188", # Triplee "880772", # approxiblue "1835379", # Cerbrus "3956566", # JamesENL "2357233", # Ms Yvette "3155639", # AlexanderOMara "462627", # Praveen Kumar "4490559", # intboolstring "1364007", # Wai Ha Lee "1699210", # bummi "563532", # Rob "5389107", # Magisch "4099593", # bhargav-rao "1542723", # Ferrybig "2025923", # Tushar "5292302", # Petter Friberg "792066", # Braiam "5666987", # Ian "3160466", # ArtOfCode "4688119", # Ashish Ahuja "3476191", # Nobody Nada "2227743", # Eric D "821878", # Ryan Bemrose "1413395", # Panta Rei "4875631", # FrankerZ "2958086", # Compass "499214", # JanDvorak "5647260", # Andrew L. "559745", # Floern "5743988", # 4castle "4622463", # angussidney "603346", # Thomas Ward "3002139", # Baum mit Augen "1863564", # QPaysTaxes "4687348", # FelixSFD "4751173", # Glorfindel "2233391", # henders "4805174", # kayess "2370483", # Machavity "1873567", # CalvT "4826457", # suraj "8242698", # user0042 "3773011", # Makyen "2858773" # Ajay Brahmakshatriya ], '111347': [ # SOBotics "3160466", # ArtOfCode "1849664", # Undo "3002139", # Baum mit Augen "3476191", # Nobody Nada "5292302", # Petter Friberg "4688119", # Ashish Ahuja "4099593", # Bhargav Rao "1743880", # Tunaki "559745", # Floern "4687348", # FelixSFD "6375113", # Bugs "4622463", # angussidney "563532", # Rob "4050842", # Thaillie "1915448" # g00glen00b ] } code_privileged_users = None smokeDetector_user_id = { charcoal_room_id: "120914", meta_tavern_room_id: "266345", socvr_room_id: "3735529", '111347': '3735529' } censored_committer_names = {"3f4ed0f38df010ce300dba362fa63a62": "Undo1"} commit = git_commit_info() if md5(commit['author'][0].encode( 'utf-8')).hexdigest() in censored_committer_names: commit['author'] = censored_committer_names[md5( commit['author'][0].encode('utf-8')).hexdigest()] commit_with_author = "%s (%s: *%s*)" % ( commit['id'], commit['author'][0] if type(commit['author']) in [list, tuple] else commit['author'], commit['message']) on_master = "HEAD detached" not in git_status() charcoal_hq = None tavern_on_the_meta = None socvr = None s = "" s_reverted = "" specialrooms = [] apiquota = -1 bodyfetcher = None se_sites = [] users_chatting = { meta_tavern_room_id: [], charcoal_room_id: [], socvr_room_id: [], '111347': [] } why_data = [] why_data_allspam = [] notifications = [] listen_to_these_if_edited = [] multiple_reporters = [] api_calls_per_site = {} standby_message = "" standby_mode = False api_request_lock = threading.Lock() num_posts_scanned = 0 post_scan_time = 0 posts_scan_stats_lock = threading.Lock() config = RawConfigParser() if os.path.isfile('config'): config.read('config') else: config.read('config.ci') latest_smokedetector_messages = { meta_tavern_room_id: [], charcoal_room_id: [], socvr_room_id: [], '111347': [] } # environ_or_none defined in helpers.py bot_name = environ_or_none("SMOKEDETECTOR_NAME") or "SmokeDetector" bot_repository = environ_or_none( "SMOKEDETECTOR_REPO") or "//github.com/Charcoal-SE/SmokeDetector" chatmessage_prefix = "[{}]({})".format(bot_name, bot_repository) site_id_dict = {} post_site_id_to_question = {} location = config.get("Config", "location") metasmoke_ws = None try: metasmoke_host = config.get("Config", "metasmoke_host") except NoOptionError: metasmoke_host = None log( 'info', "metasmoke host not found. Set it as metasmoke_host in the config file." "See https://github.com/Charcoal-SE/metasmoke.") try: metasmoke_key = config.get("Config", "metasmoke_key") except NoOptionError: metasmoke_key = "" log( 'info', "No metasmoke key found, which is okay if both are running on the same host" ) try: metasmoke_ws_host = config.get("Config", "metasmoke_ws_host") except NoOptionError: metasmoke_ws_host = "" log( 'info', "No metasmoke websocket host found, which is okay if you're anti-websocket" ) try: github_username = config.get("Config", "github_username") github_password = config.get("Config", "github_password") except NoOptionError: github_username = None github_password = None
def _read_pypirc(self): """Reads the .pypirc file.""" rc = self._get_rc_file() if os.path.exists(rc): self.announce('Using PyPI login from %s' % rc) repository = self.repository or self.DEFAULT_REPOSITORY config = RawConfigParser() config.read(rc) sections = config.sections() if 'distutils' in sections: # let's get the list of servers index_servers = config.get('distutils', 'index-servers') _servers = [ server.strip() for server in index_servers.split('\n') if server.strip() != '' ] if _servers == []: # nothing set, let's try to get the default pypi if 'pypi' in sections: _servers = ['pypi'] else: # the file is not properly defined, returning # an empty dict return {} for server in _servers: current = {'server': server} current['username'] = config.get(server, 'username') # optional params for key, default in (('repository', self.DEFAULT_REPOSITORY), ('realm', self.DEFAULT_REALM), ('password', None)): if config.has_option(server, key): current[key] = config.get(server, key) else: current[key] = default # work around people having "repository" for the "pypi" # section of their config set to the HTTP (rather than # HTTPS) URL if (server == 'pypi' and repository in (self.DEFAULT_REPOSITORY, 'pypi')): current['repository'] = self.DEFAULT_REPOSITORY return current if (current['server'] == repository or current['repository'] == repository): return current elif 'server-login' in sections: # old format server = 'server-login' if config.has_option(server, 'repository'): repository = config.get(server, 'repository') else: repository = self.DEFAULT_REPOSITORY return { 'username': config.get(server, 'username'), 'password': config.get(server, 'password'), 'repository': repository, 'server': server, 'realm': self.DEFAULT_REALM } return {}
from functions_process import check_subt_divulge, replace_xml_win from functions_picture import check_pic, add_watermark_subt from functions_requests import download_pic ######################################################################################################################## from functions_picture import add_watermark_divulge, crop_poster_baidu, crop_poster_default from functions_requests import get_search_db_html, get_db_html # main开始 print('1、若一直连不上javdb,请在ini中更新防屏蔽网址\n' '2、javdb限制搜索次数,5分钟只能搜索12次左右,然后睡眠5分钟,建议挂机整理!\n' ' 如果刚启动整理就睡眠,请检查当前网络环境能否访问javdb!\n' '4、整理FC2,fc2的信息非常非常少,大概只有个标题、卖家(片商)\n') # 读取配置文件,这个ini文件用来给用户设置 print('正在读取ini中的设置...', end='') try: config_settings = RawConfigParser() config_settings.read('【点我设置整理规则】.ini', encoding='utf-8-sig') #################################################################################################################### # 是否 收集nfo bool_nfo = True if config_settings.get("收集nfo", "是否收集nfo?") == '是' else False # 是否 跳过已存在nfo的文件夹,不整理已有nfo的文件夹 bool_skip = True if config_settings.get( "收集nfo", "是否跳过已存在nfo的文件夹?") == '是' else False # 自定义 nfo中title的格式 custom_nfo_title = config_settings.get("收集nfo", "nfo中title的格式") # 自定义 将片商等元素作为特征,因为emby不会直接在影片介绍页面上显示片商 custom_genres = config_settings.get("收集nfo", "额外将以下元素添加到特征中") # 是否 将特征保存到风格中 bool_genre = True if config_settings.get( "收集nfo", "是否将特征保存到genre?") == '是' else False
def parse_mapping(fileobj, filename=None): """Parse an extraction method mapping from a file-like object. >>> buf = StringIO(''' ... [extractors] ... custom = mypackage.module:myfunc ... ... # Python source files ... [python: **.py] ... ... # Genshi templates ... [genshi: **/templates/**.html] ... include_attrs = ... [genshi: **/templates/**.txt] ... template_class = genshi.template:TextTemplate ... encoding = latin-1 ... ... # Some custom extractor ... [custom: **/custom/*.*] ... ''') >>> method_map, options_map = parse_mapping(buf) >>> len(method_map) 4 >>> method_map[0] ('**.py', 'python') >>> options_map['**.py'] {} >>> method_map[1] ('**/templates/**.html', 'genshi') >>> options_map['**/templates/**.html']['include_attrs'] '' >>> method_map[2] ('**/templates/**.txt', 'genshi') >>> options_map['**/templates/**.txt']['template_class'] 'genshi.template:TextTemplate' >>> options_map['**/templates/**.txt']['encoding'] 'latin-1' >>> method_map[3] ('**/custom/*.*', 'mypackage.module:myfunc') >>> options_map['**/custom/*.*'] {} :param fileobj: a readable file-like object containing the configuration text to parse :see: `extract_from_directory` """ extractors = {} method_map = [] options_map = {} parser = RawConfigParser() parser._sections = OrderedDict( parser._sections) # We need ordered sections parser.read_file(fileobj, filename) for section in parser.sections(): if section == 'extractors': extractors = dict(parser.items(section)) else: method, pattern = [part.strip() for part in section.split(':', 1)] method_map.append((pattern, method)) options_map[pattern] = dict(parser.items(section)) if extractors: for idx, (pattern, method) in enumerate(method_map): if method in extractors: method = extractors[method] method_map[idx] = (pattern, method) return method_map, options_map
def __init__(self, monolithe_config, api_info): """ Initializes a _JavaSDKAPIVersionFileWriter """ super(APIVersionWriter, self).__init__(package="monolithe.generators.lang.vro") self.api_version = api_info["version"] self._api_version_string = SDKUtils.get_string_version( self.api_version) self.api_root = api_info["root"] self.api_prefix = api_info["prefix"] self.monolithe_config = monolithe_config self._output = self.monolithe_config.get_option( "output", "transformer") self._name = self.monolithe_config.get_option("name", "transformer") self._class_prefix = "" self._product_accronym = self.monolithe_config.get_option( "product_accronym") self._product_name = self.monolithe_config.get_option("product_name") self._url = self.monolithe_config.get_option("url", "transformer") self._package_prefix = self._get_package_prefix(self._url) self._package_name = self._package_prefix + ".vro." + self._name self._package_subdir = self._package_name.replace('.', '/') self.output_directory = "%s/vro" % (self._output) self.override_folder = os.path.normpath("%s/__overrides" % self.output_directory) self.fetchers_path = "/fetchers/" self.enums_path = "/enums/" self.attrs_defaults = RawConfigParser() path = "%s/vro/__attributes_defaults/attrs_defaults.ini" % self._output self.attrs_defaults.optionxform = str self.attrs_defaults.read(path) self.inventory_entities = RawConfigParser() path = "%s/vro/__attributes_defaults/inventory_entities.ini" % self._output self.inventory_entities.optionxform = str self.inventory_entities.read(path) self.workflow_attrs = RawConfigParser() path = "%s/vro/__attributes_defaults/workflow_attrs.ini" % self._output self.workflow_attrs.optionxform = str self.workflow_attrs.read(path) self.attrs_types = RawConfigParser() path = "%s/vro/__attributes_defaults/attrs_types.ini" % self._output self.attrs_types.optionxform = str self.attrs_types.read(path) self.plugin_version = self.monolithe_config.get_option( "version", "transformer") self.workflow_version = self.monolithe_config.get_option( "version", "transformer") with open("%s/vro/__code_header" % self._output, "r") as f: self.header_content = f.read()
def test_get_config_value_dtype_bool(self): config = RawConfigParser() config["section"] = {"key": "False"} val = get_config_value(config, "section", "key") self.assertEqual(val, False)
def parser(self): parser = RawConfigParser() parser.read(self.inifile) return parser
if not cfg['_MESMERIZE_PREFIX_COMMANDS'].endswith('\n'): cfg['_MESMERIZE_PREFIX_COMMANDS'] += '\n' with open(sys_cfg_file, 'w') as f: json.dump(cfg, f, indent=4) ################################################################# # Project Configuration ################################################################# proj_path = None proj_cfg = RawConfigParser(allow_no_value=True) proj_cfg['ROI_DEFS'] = {} proj_cfg['STIM_DEFS'] = {} proj_cfg.optionxform = str special = {} df_refs = {} def save_proj_config(): set_proj_special() with open(proj_path + '/config.cfg', 'w') as configfile: proj_cfg.write(configfile) def create_new_proj_config():
def __init__(self, bus: Bus, parser=RawConfigParser(defaults=DEFAULTS, strict=True)): self.parser = parser self._bus = bus self.load()
https://docs.djangoproject.com/en/2.2/topics/settings/ For the full list of settings and their values, see https://docs.djangoproject.com/en/2.2/ref/settings/ """ import os from configparser import RawConfigParser from ast import literal_eval # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # Create RawConfigParser instance and read a conf file with all that # secret and critical settings shit conf_parser = RawConfigParser() conf_parser.read(os.path.join(BASE_DIR, 'conf', 'local.conf')) # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/2.2/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = conf_parser.get('keys', 'SECRET_KEY') # SECURITY WARNING: don't run with debug turned on in production! DEBUG = conf_parser.getboolean('common', 'DEBUG', fallback=False) INTERNAL_IPS = [ '127.0.0.1', ]
def import_eds(source, node_id): eds = RawConfigParser() eds.optionxform = str if hasattr(source, "read"): fp = source else: fp = open(source) try: # Python 3 eds.read_file(fp) except AttributeError: # Python 2 eds.readfp(fp) fp.close() od = objectdictionary.ObjectDictionary() if eds.has_section("FileInfo"): od.__edsFileInfo = { opt: eds.get("FileInfo", opt) for opt in eds.options("FileInfo") } if eds.has_section("Comments"): linecount = int(eds.get("Comments", "Lines"), 0) od.comments = '\n'.join([ eds.get("Comments", "Line%i" % line) for line in range(1, linecount + 1) ]) if not eds.has_section("DeviceInfo"): logger.warn( "eds file does not have a DeviceInfo section. This section is mandatory" ) else: for rate in [10, 20, 50, 125, 250, 500, 800, 1000]: baudPossible = int( eds.get("DeviceInfo", "BaudRate_%i" % rate, fallback='0'), 0) if baudPossible != 0: od.device_information.allowed_baudrates.add(rate * 1000) for t, eprop, odprop in [ (str, "VendorName", "vendor_name"), (int, "VendorNumber", "vendor_number"), (str, "ProductName", "product_name"), (int, "ProductNumber", "product_number"), (int, "RevisionNumber", "revision_number"), (str, "OrderCode", "order_code"), (bool, "SimpleBootUpMaster", "simple_boot_up_master"), (bool, "SimpleBootUpSlave", "simple_boot_up_slave"), (bool, "Granularity", "granularity"), (bool, "DynamicChannelsSupported", "dynamic_channels_supported"), (bool, "GroupMessaging", "group_messaging"), (int, "NrOfRXPDO", "nr_of_RXPDO"), (int, "NrOfTXPDO", "nr_of_TXPDO"), (bool, "LSS_Supported", "LSS_supported"), ]: try: if t in (int, bool): setattr(od.device_information, odprop, t(int(eds.get("DeviceInfo", eprop), 0))) elif t is str: setattr(od.device_information, odprop, eds.get("DeviceInfo", eprop)) except NoOptionError: pass if eds.has_section("DeviceComissioning"): od.bitrate = int(eds.get("DeviceComissioning", "BaudRate")) * 1000 od.node_id = int(eds.get("DeviceComissioning", "NodeID"), 0) for section in eds.sections(): # Match dummy definitions match = re.match(r"^[Dd]ummy[Uu]sage$", section) if match is not None: for i in range(1, 8): key = "Dummy%04d" % i if eds.getint(section, key) == 1: var = objectdictionary.Variable(key, i, 0) var.data_type = i var.access_type = "const" od.add_object(var) # Match indexes match = re.match(r"^[0-9A-Fa-f]{4}$", section) if match is not None: index = int(section, 16) name = eds.get(section, "ParameterName") try: object_type = int(eds.get(section, "ObjectType"), 0) except NoOptionError: # DS306 4.6.3.2 object description # If the keyword ObjectType is missing, this is regarded as # "ObjectType=0x7" (=VAR). object_type = VAR try: storage_location = eds.get(section, "StorageLocation") except NoOptionError: storage_location = None if object_type in (VAR, DOMAIN): var = build_variable(eds, section, node_id, index) od.add_object(var) elif object_type == ARR and eds.has_option(section, "CompactSubObj"): arr = objectdictionary.Array(name, index) last_subindex = objectdictionary.Variable( "Number of entries", index, 0) last_subindex.data_type = objectdictionary.UNSIGNED8 arr.add_member(last_subindex) arr.add_member(build_variable(eds, section, node_id, index, 1)) arr.storage_location = storage_location od.add_object(arr) elif object_type == ARR: arr = objectdictionary.Array(name, index) arr.storage_location = storage_location od.add_object(arr) elif object_type == RECORD: record = objectdictionary.Record(name, index) record.storage_location = storage_location od.add_object(record) continue # Match subindexes match = re.match(r"^([0-9A-Fa-f]{4})[S|s]ub([0-9A-Fa-f]+)$", section) if match is not None: index = int(match.group(1), 16) subindex = int(match.group(2), 16) entry = od[index] if isinstance(entry, (objectdictionary.Record, objectdictionary.Array)): var = build_variable(eds, section, node_id, index, subindex) entry.add_member(var) # Match [index]Name match = re.match(r"^([0-9A-Fa-f]{4})Name", section) if match is not None: index = int(match.group(1), 16) num_of_entries = int(eds.get(section, "NrOfEntries")) entry = od[index] # For CompactSubObj index 1 is were we find the variable src_var = od[index][1] for subindex in range(1, num_of_entries + 1): var = copy_variable(eds, section, subindex, src_var) if var is not None: entry.add_member(var) return od
'data_column_fixid': '0', 'data_column_catid': '1', 'data_column_utcdatetime': '4', 'data_column_utmy': '6', 'data_column_utmx': '7', 'radius': '200', 'time_cutoff': '144', 'minimum_count': '0', 'minimum_stay': '0', 'start_date': '0', 'end_date': '0', 'dot_size': '4', 'perimeter_resolution': '9' } config = RawConfigParser(fallback_values) config.read(configfilepath) cfg_datafile_path = config.get('Global_Settings', 'datafile_path') cfg_outdir_path = config.get('Global_Settings', 'outdir_path') cfg_data_column_fixid = config.get('Global_Settings', 'data_column_fixid') cfg_data_column_catid = config.get('Global_Settings', 'data_column_catid') cfg_data_column_utcdatetime = config.get('Global_Settings', 'data_column_utcdatetime') cfg_data_column_utmy = config.get('Global_Settings', 'data_column_utmy') cfg_data_column_utmx = config.get('Global_Settings', 'data_column_utmx') cfg_cluster_radius = config.get('Cluster_Settings', 'radius') cfg_cluster_time_cutoff = config.get('Cluster_Settings', 'time_cutoff') cfg_cluster_minimum_count = config.get('Cluster_Settings', 'minimum_count') cfg_cluster_minimum_stay = config.get('Cluster_Settings', 'minimum_stay')
def set_config(): sys_para = sys.argv file_path = os.path.split(sys_para[0])[0] gui = False if platform.uname()[0] == 'Windows': # Win默认打开 gui = True if platform.uname()[0] == 'Linux': # Linux 默认关闭 gui = False if '--gui' in sys.argv: # 指定 gui 模式 gui = True if '--nogui' in sys.argv: # 带 nogui 就覆盖前面Win打开要求 gui = False config_file = os.path.join(file_path, 's3_download_config.ini') # If no config file, read the default config if not os.path.exists(config_file): config_file += '.default' print("No customized config, use the default config") cfg = ConfigParser() print(f'Reading config file: {config_file}') try: global SrcBucket, S3Prefix, SrcFileIndex, SrcProfileName, DesDir, MaxRetry, MaxThread, MaxParallelFile, LoggingLevel cfg.read(config_file, encoding='utf-8-sig') SrcBucket = cfg.get('Basic', 'SrcBucket') S3Prefix = cfg.get('Basic', 'S3Prefix') SrcFileIndex = cfg.get('Basic', 'SrcFileIndex') SrcProfileName = cfg.get('Basic', 'SrcProfileName') DesDir = cfg.get('Basic', 'DesDir') Megabytes = 1024 * 1024 ChunkSize = cfg.getint('Advanced', 'ChunkSize') * Megabytes MaxRetry = cfg.getint('Advanced', 'MaxRetry') MaxThread = cfg.getint('Advanced', 'MaxThread') MaxParallelFile = cfg.getint('Advanced', 'MaxParallelFile') LoggingLevel = cfg.get('Advanced', 'LoggingLevel') except Exception as e: print("ERR loading s3_download_config.ini", str(e)) input('PRESS ENTER TO QUIT') sys.exit(0) if gui: # For GUI from tkinter import Tk, filedialog, END, StringVar, BooleanVar, messagebox from tkinter.ttk import Combobox, Label, Button, Entry, Spinbox, Checkbutton # get profile name list in ./aws/credentials pro_conf = RawConfigParser() pro_path = os.path.join(os.path.expanduser("~"), ".aws") cre_path = os.path.join(pro_path, "credentials") if os.path.exists(cre_path): pro_conf.read(cre_path) profile_list = pro_conf.sections() else: print( f"There is no aws_access_key in {cre_path}, please input for S3 Bucket: " ) os.mkdir(pro_path) aws_access_key_id = input('aws_access_key_id: ') aws_secret_access_key = input('aws_secret_access_key: ') region = input('region: ') pro_conf.add_section('default') pro_conf['default']['aws_access_key_id'] = aws_access_key_id pro_conf['default'][ 'aws_secret_access_key'] = aws_secret_access_key pro_conf['default']['region'] = region profile_list = ['default'] with open(cre_path, 'w') as f: print(f"Saving credentials to {cre_path}") pro_conf.write(f) # Click Select Folder def browse_folder(): local_dir = filedialog.askdirectory( initialdir=os.path.dirname(__file__)) url_txt.delete(0, END) url_txt.insert(0, local_dir) file_txt.delete(0, END) file_txt.insert(0, "*") # Finsih browse folder # Click List Buckets def ListBuckets(*args): SrcProfileName = SrcProfileName_txt.get() client = Session(profile_name=SrcProfileName).client('s3') bucket_list = [] try: response = client.list_buckets() if 'Buckets' in response: bucket_list = [b['Name'] for b in response['Buckets']] except Exception as e: messagebox.showerror( 'Error', f'Failt to List buckets. \n' f'Please verify your aws_access_key of profile: [{SrcProfileName}]\n' f'{str(e)}') bucket_list = ['CAN_NOT_GET_BUCKET_LIST'] SrcBucket_txt['values'] = bucket_list SrcBucket_txt.current(0) # Finish ListBuckets # Click List Prefix def ListPrefix(*args): SrcProfileName = SrcProfileName_txt.get() client = Session(profile_name=SrcProfileName).client('s3') prefix_list = [] this_bucket = SrcBucket_txt.get() max_get = 100 try: response = client.list_objects_v2( Bucket=this_bucket, Delimiter='/' ) # Only get the max 1000 prefix for simply list if 'CommonPrefixes' in response: prefix_list = [ c['Prefix'] for c in response['CommonPrefixes'] ] if not prefix_list: messagebox.showinfo( 'Message', f'There is no "/" Prefix in: {this_bucket}') if response['IsTruncated']: messagebox.showinfo( 'Message', f'More than {max_get} Prefix, cannot fully list here.') except Exception as e: messagebox.showinfo( 'Error', f'Cannot get prefix list from bucket: {this_bucket}, {str(e)}' ) S3Prefix_txt['values'] = prefix_list S3Prefix_txt.current(0) # Finish list prefix def browse_file(*args): SrcProfileName = SrcProfileName_txt.get() S3Prefix = S3Prefix_txt.get() client = Session(profile_name=SrcProfileName).client('s3') file_list = [] this_bucket = SrcBucket_txt.get() max_get = 100 try: response = client.list_objects_v2( Bucket=this_bucket, Prefix=str(PurePosixPath(S3Prefix)) + '/', Delimiter='/' ) # Only get the max 1000 files for simply list # For delete prefix in des_prefix if S3Prefix == '' or S3Prefix == '/': # 目的bucket没有设置 Prefix dp_len = 0 else: # 目的bucket的 "prefix/"长度 dp_len = len(str(PurePosixPath(S3Prefix))) + 1 if 'Contents' in response: file_list = [ c['Key'][dp_len:] for c in response['Contents'] ] # 去掉Prefix if not file_list: messagebox.showinfo( 'Message', f'There is no files in s3://{this_bucket}/{S3Prefix}') if response['IsTruncated']: messagebox.showinfo( 'Message', f'More than {max_get} files, cannot fully list here.') except Exception as e: messagebox.showinfo( 'Error', f'Cannot get file list from bucket s3://{this_bucket}/{S3Prefix}, {str(e)}' ) file_txt['values'] = file_list file_txt.current(0) # Finish list files # Click START button def close(): window.withdraw() ok = messagebox.askokcancel( 'Start downloading job', f'DOWNLOAD FROM s3://{SrcBucket_txt.get()}/{S3Prefix_txt.get()}\n' f'TO LOCAL {url_txt.get()}\n' f'Click OK to START') if not ok: window.deiconify() return window.quit() return # Finish close() # Start GUI window = Tk() window.title( "LONGBOW - AMAZON S3 DOWNLOAD TOOL WITH BREAK-POINT RESUMING") window.geometry('705x350') window.configure(background='#ECECEC') window.protocol("WM_DELETE_WINDOW", sys.exit) Label(window, text="S3 Bucket").grid(column=0, row=1, sticky='w', padx=2, pady=2) SrcBucket_txt = Combobox(window, width=48) SrcBucket_txt.grid(column=1, row=1, sticky='w', padx=2, pady=2) SrcBucket_txt['values'] = SrcBucket SrcBucket_txt.current(0) Button(window, text="List Buckets", width=10, command=ListBuckets) \ .grid(column=2, row=1, sticky='w', padx=2, pady=2) Label(window, text="S3 Prefix").grid(column=0, row=2, sticky='w', padx=2, pady=2) S3Prefix_txt = Combobox(window, width=48) S3Prefix_txt.grid(column=1, row=2, sticky='w', padx=2, pady=2) S3Prefix_txt['values'] = S3Prefix if S3Prefix != '': S3Prefix_txt.current(0) Button(window, text="List Prefix", width=10, command=ListPrefix) \ .grid(column=2, row=2, sticky='w', padx=2, pady=2) Label(window, text="Filename or *").grid(column=0, row=3, sticky='w', padx=2, pady=2) file_txt = Combobox(window, width=48) file_txt.grid(column=1, row=3, sticky='w', padx=2, pady=2) file_txt['values'] = SrcFileIndex if SrcFileIndex != '': file_txt.current(0) Button(window, text="Select File", width=10, command=browse_file) \ .grid(column=2, row=3, sticky='w', padx=2, pady=2) Label(window, text="AWS Profile").grid(column=0, row=4, sticky='w', padx=2, pady=2) SrcProfileName_txt = Combobox(window, width=15, state="readonly") SrcProfileName_txt['values'] = tuple(profile_list) SrcProfileName_txt.grid(column=1, row=4, sticky='w', padx=2, pady=2) if SrcProfileName in profile_list: position = profile_list.index(SrcProfileName) SrcProfileName_txt.current(position) else: SrcProfileName_txt.current(0) SrcProfileName = SrcProfileName_txt.get() SrcProfileName_txt.bind("<<ComboboxSelected>>", ListBuckets) Label(window, text="Folder").grid(column=0, row=5, sticky='w', padx=2, pady=2) url_txt = Entry(window, width=50) url_txt.grid(column=1, row=5, sticky='w', padx=2, pady=2) url_btn = Button(window, text="Select Folder", width=10, command=browse_folder) url_btn.grid(column=2, row=5, sticky='w', padx=2, pady=2) url_txt.insert(0, DesDir) Label(window, text="MaxThread/File").grid(column=0, row=6, sticky='w', padx=2, pady=2) if MaxThread < 1 or MaxThread > 100: MaxThread = 5 var_t = StringVar() var_t.set(str(MaxThread)) MaxThread_txt = Spinbox(window, from_=1, to=100, width=15, textvariable=var_t) MaxThread_txt.grid(column=1, row=6, sticky='w', padx=2, pady=2) Label(window, text="MaxParallelFile").grid(column=0, row=7, sticky='w', padx=2, pady=2) if MaxParallelFile < 1 or MaxParallelFile > 100: MaxParallelFile = 5 var_f = StringVar() var_f.set(str(MaxParallelFile)) MaxParallelFile_txt = Spinbox(window, from_=1, to=100, width=15, textvariable=var_f) MaxParallelFile_txt.grid(column=1, row=7, sticky='w', padx=2, pady=2) save_config = BooleanVar() save_config.set(True) save_config_txt = Checkbutton(window, text="Save to s3_download_config.ini", var=save_config) save_config_txt.grid(column=1, row=9, padx=2, pady=2) Button(window, text="Start Download", width=15, command=close).grid(column=1, row=10, padx=5, pady=5) window.mainloop() DesDir = url_txt.get() SrcFileIndex = file_txt.get() SrcBucket = SrcBucket_txt.get() S3Prefix = S3Prefix_txt.get() SrcProfileName = SrcProfileName_txt.get() MaxThread = int(MaxThread_txt.get()) MaxParallelFile = int(MaxParallelFile_txt.get()) if save_config: cfg['Basic']['SrcBucket'] = SrcBucket cfg['Basic']['S3Prefix'] = S3Prefix cfg['Basic']['SrcFileIndex'] = SrcFileIndex cfg['Basic']['SrcProfileName'] = SrcProfileName cfg['Basic']['DesDir'] = DesDir cfg['Advanced']['MaxThread'] = str(MaxThread) cfg['Advanced']['MaxParallelFile'] = str(MaxParallelFile) config_file = os.path.join(file_path, 's3_download_config.ini') with codecs.open(config_file, 'w', 'utf-8') as f: cfg.write(f) print(f"Save config to {config_file}") # GUI window finish if S3Prefix == '/': S3Prefix = '' # Finish set_config() return ChunkSize
def read_config(self, path): config = RawConfigParser() config.optionxform = lambda s: s config.read(path) return config
def write_file(self): here = "config.write_file" config_write = RawConfigParser() section = "Debug" config_write.add_section(section) config_write.set(section, 'debug_reread_config', self.debug_reread_config) config_write.set(section, 'debug_flag_1', self.debug_flag_1) config_write.set(section, 'debug_flag_2', self.debug_flag_2) config_write.set(section, 'debug_flag_ftp', self.debug_flag_ftp) section = "Scan" config_write.add_section(section) config_write.set(section, 'scan_delay', self.scan_delay) config_write.set(section, 'max_scans', self.max_scans) section = "Log" config_write.add_section(section) config_write.set(section, 'log_directory', self.log_directory) config_write.set(section, 'local_dir_www', self.local_dir_www) config_write.set(section, 'log_buffer_flag', self.log_buffer_flag) config_write.set(section, 'text_buffer_length', self.text_buffer_length) section = "Ftp" config_write.add_section(section) config_write.set(section, 'ftp_creds_filename', self.ftp_creds_filename) config_write.set(section, 'ftp_log_max_count', self.ftp_log_max_count) section = "Heating_Fan" config_write.add_section(section) config_write.set(section, 'heat_max_temp', self.heat_max_temp) config_write.set(section, 'heat_min_temp', self.heat_min_temp) config_write.set(section, 'heat_max_speed', self.heat_max_speed) config_write.set(section, 'heat_min_speed', self.heat_min_speed) config_write.set(section, 'heat_max_freq', self.heat_max_freq) config_write.set(section, 'heat_min_freq', self.heat_min_freq) section = "Sauna" config_write.add_section(section) config_write.set(section, 'sauna_max_temp', self.sauna_max_temp) config_write.set(section, 'sauna_min_temp', self.sauna_min_temp) config_write.set(section, 'sauna_max_speed', self.sauna_max_speed) config_write.set(section, 'sauna_min_speed', self.sauna_min_speed) config_write.set(section, 'sauna_max_freq', self.sauna_max_freq) config_write.set(section, 'sauna_min_freq', self.sauna_min_freq) config_write.set(section, 'sauna_GPIO_port', self.sauna_GPIO_port) config_write.set(section, 'sauna_brightness', self.sauna_brightness) section = "Power_Log" config_write.add_section(section) config_write.set(section, 'adc_scan_size', self.adc_scan_size) config_write.set(section, 'adc_target_scan_msec', self.adc_target_scan_msec) config_write.set(section, 'adc_channel', self.adc_channel) config_write.set(section, 'adc_default_gain ', self.adc_default_gain) config_write.set(section, 'adc_top_limit', self.adc_top_limit) config_write.set(section, 'adc_bottom_limit', self.adc_bottom_limit) config_write.set(section, 'adc_input_offset_mv', self.adc_input_offset_mv) config_write.set(section, 'adc_input_amp_gain', self.adc_input_amp_gain) config_write.set(section, 'adc_CT_ratio', self.adc_CT_ratio) config_write.set(section, 'adc_CT_resister', self.adc_CT_resister) # Writing our configuration file to 'self.config_filename' pr(self.debug_flag_1, here, "ready to write new config file with default values: ", self.config_filename) with open(self.config_filename, 'w+') as configfile: config_write.write(configfile) return 0
class GlobalVars: on_windows = 'windows' in platform.platform().lower() false_positives = [] whitelisted_users = set() blacklisted_users = dict() blacklisted_usernames = [] blacklisted_websites = [] blacklisted_numbers = [] watched_numbers = [] blacklisted_numbers_normalized = None watched_numbers_normalized = None bad_keywords = [] watched_keywords = {} ignored_posts = [] auto_ignored_posts = [] startup_utc_date = datetime.utcnow() startup_utc = startup_utc_date.strftime("%H:%M:%S") latest_questions = [] api_backoff_time = 0 deletion_watcher = None not_privileged_warning = \ "You are not a privileged user. Please see " \ "[the privileges wiki page](https://charcoal-se.org/smokey/Privileges) for " \ "information on what privileges are and what is expected of privileged users." experimental_reasons = { # Don't widely report these "potentially bad keyword in answer", "potentially bad keyword in body", "potentially bad keyword in title", "potentially bad keyword in username", "potentially bad NS for domain in title", "potentially bad NS for domain in body", "potentially bad NS for domain in answer", "potentially bad ASN for hostname in title", "potentially bad ASN for hostname in body", "potentially bad ASN for hostname in answer", "potentially bad IP for hostname in title", "potentially bad IP for hostname in body", "potentially bad IP for hostname in answer", "potentially problematic NS configuration in title", "potentially problematic NS configuration in body", "potentially problematic NS configuration in answer", "toxic body detected", "toxic answer detected", } parser = HTMLParser() parser.unescape = unescape code_privileged_users = None # these are loaded in GlobalVars.reload() commit = None commit_with_author = None on_branch = None s = "" s_reverted = "" s_norestart_blacklists = "" s_norestart_findspam = "" apiquota = -1 bodyfetcher = None cookies = {} se_sites = [] why_data = [] notifications = [] listen_to_these_if_edited = [] multiple_reporters = [] api_calls_per_site = {} reason_weights = {} metasmoke_ids = {} standby_message = "" standby_mode = False no_se_activity_scan = False api_request_lock = threading.Lock( ) # Get this lock before making API requests apiquota_rw_lock = threading.Lock( ) # Get this lock before reading/writing apiquota class PostScanStat: """ Tracking post scanning data """ num_posts_scanned = 0 post_scan_time = 0 rw_lock = threading.Lock() @staticmethod def add_stat(posts_scanned, scan_time): """ Adding post scanning data """ with GlobalVars.PostScanStat.rw_lock: GlobalVars.PostScanStat.num_posts_scanned += posts_scanned GlobalVars.PostScanStat.post_scan_time += scan_time @staticmethod def get_stat(): """ Getting post scanning statistics """ with GlobalVars.PostScanStat.rw_lock: posts_scanned = GlobalVars.PostScanStat.num_posts_scanned scan_time = GlobalVars.PostScanStat.post_scan_time if scan_time == 0: posts_per_second = None else: posts_per_second = posts_scanned / scan_time return (posts_scanned, scan_time, posts_per_second) @staticmethod def reset_stat(): """ Resetting post scanning data """ with GlobalVars.PostScanStat.rw_lock: GlobalVars.PostScanStat.num_posts_scanned = 0 GlobalVars.PostScanStat.post_scan_time = 0 config_parser = RawConfigParser() if os.path.isfile('config') and "pytest" not in sys.modules: config_parser.read('config') else: config_parser.read('config.ci') config = config_parser["Config"] # It's a collections.OrderedDict now site_id_dict = {} post_site_id_to_question = {} location = config.get("location", "Continuous Integration") class MSStatus: """ Tracking metasmoke status """ ms_is_up = True counter = 0 rw_lock = threading.Lock() @staticmethod def set_up(): """ Set metasmoke status to up """ # Private to metasmoke.py with GlobalVars.MSStatus.rw_lock: GlobalVars.MSStatus.ms_is_up = True @staticmethod def set_down(): """ Set metasmoke status to down """ # Private to metasmoke.py with GlobalVars.MSStatus.rw_lock: GlobalVars.MSStatus.ms_is_up = False @staticmethod def is_up(): """ Query if metasmoke status is up """ with GlobalVars.MSStatus.rw_lock: current_ms_status = GlobalVars.MSStatus.ms_is_up return current_ms_status @staticmethod def is_down(): """ Query if metasmoke status is down """ return not GlobalVars.MSStatus.is_up() # Why implement failed() and succeeded() here, as they will only be called in metasmoke.py? # Because get_failure_count() need to be exposed to global, so it is more convenient # to implement failed() and succeeded() here. @staticmethod def failed(): """ Indicate a metasmoke connection failure """ with GlobalVars.MSStatus.rw_lock: GlobalVars.MSStatus.counter += 1 @staticmethod def succeeded(): """ Indicate a metasmoke connection success """ with GlobalVars.MSStatus.rw_lock: GlobalVars.MSStatus.counter = 0 @staticmethod def get_failure_count(): """ Get consecutive metasmoke connection failure count """ with GlobalVars.MSStatus.rw_lock: failure_count = GlobalVars.MSStatus.counter return failure_count @staticmethod def reset_ms_status(): """ Reset class GlobalVars.MSStatus to default values """ with GlobalVars.MSStatus.rw_lock: GlobalVars.MSStatus.ms_is_up = True GlobalVars.MSStatus.counter = 0 chatexchange_u = config.get("ChatExchangeU") chatexchange_p = config.get("ChatExchangeP") metasmoke_host = config.get("metasmoke_host") metasmoke_key = config.get("metasmoke_key") metasmoke_ws_host = config.get("metasmoke_ws_host") git_name = config.get("git_username", "SmokeDetector") git_email = config.get("git_useremail", "*****@*****.**") github_username = config.get("github_username") github_password = config.get("github_password") github_access_token = config.get("github_access_token") perspective_key = config.get("perspective_key") flovis_host = config.get("flovis_host") flovis = None # Miscellaneous log_time_format = config.get("log_time_format", "%H:%M:%S") # Blacklist privileged users from config se_blacklisters = regex.sub(r"[^\d,]", "", config.get("se_blacklisters", "")).split(",") mse_blacklisters = regex.sub(r"[^\d,]", "", config.get("mse_blacklisters", "")).split(",") so_blacklisters = regex.sub(r"[^\d,]", "", config.get("so_blacklisters", "")).split(",") # Create a set of blacklisters equivalent to what's used in code_privileged_users. config_blacklisters = set() for id in se_blacklisters: if id: config_blacklisters.add(("stackexchange.com", int(id))) for id in mse_blacklisters: if id: config_blacklisters.add(("meta.stackexchange.com", int(id))) for id in so_blacklisters: if id: config_blacklisters.add(("stackoverflow.com", int(id))) # environ_or_none replaced by os.environ.get (essentially dict.get) bot_name = os.environ.get("SMOKEDETECTOR_NAME", git_name) bot_repo_slug = os.environ.get("SMOKEDETECTOR_REPO", git_user_repo) bot_repository = "//github.com/{}".format(bot_repo_slug) chatmessage_prefix = "[{}]({})".format(bot_name, bot_repository) valid_content = """This is a totally valid post that should never be caught. Any blacklist or watchlist item that triggers on this item should be avoided. java.io.BbbCccDddException: nothing wrong found. class Safe { perfect valid code(int float &#%$*v a b c =+ /* - 0 1 2 3 456789.EFGQ} English 中文Français Español Português Italiano Deustch ~@#%*-_/'()?!:;" vvv kkk www sss ttt mmm absolute std::adjacent_find (power).each do |s| bbb end ert zal l gsopsq kdowhs@ xjwk* %_sooqmzb xjwpqpxnf. Please don't blacklist disk-partition.com, it's a valid domain (though it also gets spammed rather frequently).""" # noqa: E501 @classmethod def reload(cls): cls.commit = commit = git_commit_info() cls.commit_with_author = "`{}` ({}: {})".format( commit.id, commit.author, commit.message) # We don't want to escape `[` and `]` when they are within code. split_commit_with_author = cls.commit_with_author.split('`') split_length = len(split_commit_with_author) for index in range(0, split_length, 2): split_commit_with_author[index] = split_commit_with_author[ index].replace('[', '\\[').replace(']', '\\]') # There's not an even number of ` characters, so the parsing hack failed, but we assume the last one needs # escaping. if not split_length % 2: split_commit_with_author[-1] = split_commit_with_author[ -1].replace('[', '\\[').replace(']', '\\]') cls.commit_with_author_escaped = '`'.join(split_commit_with_author) cls.on_branch = git_ref() cls.s = "[ {} ] SmokeDetector started at [rev {}]({}/commit/{}) (running on {}, Python {})".format( cls.chatmessage_prefix, cls.commit_with_author_escaped, cls.bot_repository, cls.commit.id, cls.location, platform.python_version()) cls.s_reverted = \ "[ {} ] SmokeDetector started in [reverted mode](" \ "https://charcoal-se.org/smokey/SmokeDetector-Statuses#reverted-mode) " \ "at [rev {}]({}/commit/{}) (running on {})".format( cls.chatmessage_prefix, cls.commit_with_author_escaped, cls.bot_repository, cls.commit.id, cls.location) cls.s_norestart_blacklists = \ "[ {} ] Blacklists reloaded at [rev {}]({}/commit/{}) (running on {})".format( cls.chatmessage_prefix, cls.commit_with_author_escaped, cls.bot_repository, cls.commit.id, cls.location) cls.s_norestart_findspam = \ "[ {} ] FindSpam module reloaded at [rev {}]({}/commit/{}) (running on {})".format( cls.chatmessage_prefix, cls.commit_with_author_escaped, cls.bot_repository, cls.commit.id, cls.location) cls.standby_message = \ "[ {} ] SmokeDetector started in [standby mode](" \ "https://charcoal-se.org/smokey/SmokeDetector-Statuses#standby-mode) " \ "at [rev {}]({}/commit/{}) (running on {})".format( cls.chatmessage_prefix, cls.commit_with_author_escaped, cls.bot_repository, cls.commit.id, cls.location)
def __init__(self, path): config = RawConfigParser() config.optionxform = lambda s: s if path: # Test support config.read(path) self.config = config
class GlobalVars: on_windows = 'windows' in platform.platform().lower() false_positives = [] whitelisted_users = set() blacklisted_users = dict() blacklisted_usernames = [] blacklisted_websites = [] blacklisted_numbers = [] watched_numbers = [] blacklisted_numbers_normalized = None watched_numbers_normalized = None bad_keywords = [] watched_keywords = {} ignored_posts = [] auto_ignored_posts = [] startup_utc_date = datetime.utcnow() startup_utc = startup_utc_date.strftime("%H:%M:%S") latest_questions = [] api_backoff_time = 0 deletion_watcher = None metasmoke_last_ping_time = datetime.now() not_privileged_warning = \ "You are not a privileged user. Please see " \ "[the privileges wiki page](https://charcoal-se.org/smokey/Privileges) for " \ "information on what privileges are and what is expected of privileged users." experimental_reasons = { # Don't widely report these "potentially bad keyword in answer", "potentially bad keyword in body", "potentially bad keyword in title", "potentially bad keyword in username", "potentially bad NS for domain in title", "potentially bad NS for domain in body", "potentially bad NS for domain in answer", "potentially bad ASN for hostname in title", "potentially bad ASN for hostname in body", "potentially bad ASN for hostname in answer", "potentially problematic NS configuration in title", "potentially problematic NS configuration in body", "potentially problematic NS configuration in answer", "toxic body detected", "toxic answer detected", } parser = HTMLParser() parser.unescape = unescape code_privileged_users = None # these are loaded in GlobalVars.reload() commit = None commit_with_author = None on_master = None s = "" s_reverted = "" s_norestart = "" s_norestart2 = "" apiquota = -1 bodyfetcher = None cookies = {} se_sites = [] why_data = [] notifications = [] listen_to_these_if_edited = [] multiple_reporters = [] api_calls_per_site = {} reason_weights = {} metasmoke_ids = {} standby_message = "" standby_mode = False api_request_lock = threading.Lock() num_posts_scanned = 0 post_scan_time = 0 posts_scan_stats_lock = threading.Lock() config_parser = RawConfigParser() if os.path.isfile('config') and "pytest" not in sys.modules: config_parser.read('config') log('debug', "Configuration loaded from \"config\"") else: config_parser.read('config.ci') if "pytest" in sys.modules and os.path.isfile('config'): # Another config found while running in pytest log('debug', "Running in pytest, force load config from \"config.ci\"") else: log('debug', "Configuration loaded from \"config.ci\"") config = config_parser["Config"] # It's a collections.OrderedDict now # environ_or_none replaced by os.environ.get (essentially dict.get) bot_name = os.environ.get("SMOKEDETECTOR_NAME", "SmokeDetector") bot_repo_slug = os.environ.get("SMOKEDETECTOR_REPO", "Charcoal-SE/SmokeDetector") bot_repository = "//github.com/{}".format(bot_repo_slug) chatmessage_prefix = "[{}]({})".format(bot_name, bot_repository) site_id_dict = {} post_site_id_to_question = {} location = config.get("location", "Continuous Integration") metasmoke_ws = None metasmoke_down = False metasmoke_failures = 0 # Consecutive count, not cumulative chatexchange_u = config.get("ChatExchangeU") chatexchange_p = config.get("ChatExchangeP") metasmoke_host = config.get("metasmoke_host") metasmoke_key = config.get("metasmoke_key") metasmoke_ws_host = config.get("metasmoke_ws_host") github_username = config.get("github_username") github_password = config.get("github_password") perspective_key = config.get("perspective_key") flovis_host = config.get("flovis_host") flovis = None # Miscellaneous log_time_format = config.get("log_time_format", "%H:%M:%S") valid_content = """This is a totally valid post that should never be caught. Any blacklist or watchlist item that triggers on this item should be avoided. java.io.BbbCccDddException: nothing wrong found. class Safe { perfect valid code(int float &#%$*v a b c =+ /* - 0 1 2 3 456789.EFGQ} English 中文Français Español Português Italiano Deustch ~@#%*-_/'()?!:;" vvv kkk www sss ttt mmm absolute std::adjacent_find (power).each do |s| bbb end ert zal l gsopsq kdowhs@ xjwk* %_sooqmzb xjwpqpxnf. Please don't blacklist disk-partition.com, it's a valid domain (though it also gets spammed rather frequently).""" # noqa: E501 @staticmethod def reload(): commit = git_commit_info() GlobalVars.commit = commit GlobalVars.commit_with_author = "`{}` ({}: {})".format( commit['id'], commit['author'], commit['message']) GlobalVars.on_master = git_ref() GlobalVars.s = "[ {} ] SmokeDetector started at [rev {}]({}/commit/{}) (running on {}, Python {})".format( GlobalVars.chatmessage_prefix, GlobalVars.commit_with_author, GlobalVars.bot_repository, GlobalVars.commit['id'], GlobalVars.location, platform.python_version()) GlobalVars.s_reverted = \ "[ {} ] SmokeDetector started in [reverted mode](" \ "https://charcoal-se.org/smokey/SmokeDetector-Statuses#reverted-mode) " \ "at [rev {}]({}/commit/{}) (running on {})".format( GlobalVars.chatmessage_prefix, GlobalVars.commit_with_author, GlobalVars.bot_repository, GlobalVars.commit['id'], GlobalVars.location) GlobalVars.s_norestart = "[ {} ] Blacklists reloaded at [rev {}]({}/commit/{}) (running on {})".format( GlobalVars.chatmessage_prefix, GlobalVars.commit_with_author, GlobalVars.bot_repository, GlobalVars.commit['id'], GlobalVars.location) GlobalVars.s_norestart2 = "[ {} ] FindSpam module reloaded at [rev {}]({}/commit/{}) (running on {})".format( GlobalVars.chatmessage_prefix, GlobalVars.commit_with_author, GlobalVars.bot_repository, GlobalVars.commit['id'], GlobalVars.location) GlobalVars.standby_message = \ "[ {} ] SmokeDetector started in [standby mode](" \ "https://charcoal-se.org/smokey/SmokeDetector-Statuses#standby-mode) " \ "at [rev {}]({}/commit/{}) (running on {})".format( GlobalVars.chatmessage_prefix, GlobalVars.commit_with_author, GlobalVars.bot_repository, GlobalVars.commit['id'], GlobalVars.location) log('debug', "GlobalVars loaded")
def main(original_args=None): positionals, args = split_args_in_optional_and_positional( sys.argv[1:] if original_args is None else original_args) if len(positionals[1:]) > 2: warnings.warn( "Giving multiple files on the command line will be deprecated, please use [bumpversion:file:...] in a config file.", PendingDeprecationWarning) parser1 = argparse.ArgumentParser(add_help=False) parser1.add_argument( '--config-file', metavar='FILE', default=argparse.SUPPRESS, required=False, help= 'Config file to read most of the variables from (default: .bumpversion.cfg)' ) parser1.add_argument('--verbose', action='count', default=0, help='Print verbose logging to stderr', required=False) parser1.add_argument('--list', action='store_true', default=False, help='List machine readable information', required=False) parser1.add_argument('--allow-dirty', action='store_true', default=False, help="Don't abort if working directory is dirty", required=False) known_args, remaining_argv = parser1.parse_known_args(args) logformatter = logging.Formatter('%(message)s') if len(logger.handlers) == 0: ch = logging.StreamHandler(sys.stderr) ch.setFormatter(logformatter) logger.addHandler(ch) if len(logger_list.handlers) == 0: ch2 = logging.StreamHandler(sys.stdout) ch2.setFormatter(logformatter) logger_list.addHandler(ch2) if known_args.list: logger_list.setLevel(1) log_level = { 0: logging.WARNING, 1: logging.INFO, 2: logging.DEBUG, }.get(known_args.verbose, logging.DEBUG) logger.setLevel(log_level) logger.debug("Starting {}".format(DESCRIPTION)) defaults = {} vcs_info = {} for vcs in VCS: if vcs.is_usable(): vcs_info.update(vcs.latest_tag_info()) if 'current_version' in vcs_info: defaults['current_version'] = vcs_info['current_version'] config = RawConfigParser('') # don't transform keys to lowercase (which would be the default) config.optionxform = lambda option: option config.add_section('bumpversion') explicit_config = hasattr(known_args, 'config_file') if explicit_config: config_file = known_args.config_file elif not os.path.exists('.bumpversion.cfg') and \ os.path.exists('setup.cfg'): config_file = 'setup.cfg' else: config_file = '.bumpversion.cfg' config_file_exists = os.path.exists(config_file) part_configs = {} files = [] if config_file_exists: logger.info("Reading config file {}:".format(config_file)) logger.info(io.open(config_file, 'rt', encoding='utf-8').read()) config.readfp(io.open(config_file, 'rt', encoding='utf-8')) log_config = StringIO() config.write(log_config) if 'files' in dict(config.items("bumpversion")): warnings.warn( "'files =' configuration is will be deprecated, please use [bumpversion:file:...]", PendingDeprecationWarning) defaults.update(dict(config.items("bumpversion"))) for listvaluename in ("serialize", ): try: value = config.get("bumpversion", listvaluename) defaults[listvaluename] = list( filter(None, (x.strip() for x in value.splitlines()))) except NoOptionError: pass # no default value then ;) for boolvaluename in ("commit", "tag", "dry_run"): try: defaults[boolvaluename] = config.getboolean( "bumpversion", boolvaluename) except NoOptionError: pass # no default value then ;) for section_name in config.sections(): section_name_match = re.compile( "^bumpversion:(file|part):(.+)").match(section_name) if not section_name_match: continue section_prefix, section_value = section_name_match.groups() section_config = dict(config.items(section_name)) if section_prefix == "part": ThisVersionPartConfiguration = NumericVersionPartConfiguration if 'values' in section_config: section_config['values'] = list( filter( None, (x.strip() for x in section_config['values'].splitlines()))) ThisVersionPartConfiguration = ConfiguredVersionPartConfiguration part_configs[section_value] = ThisVersionPartConfiguration( **section_config) elif section_prefix == "file": filename = section_value if 'serialize' in section_config: section_config['serialize'] = list( filter( None, (x.strip() for x in section_config['serialize'].splitlines() ))) section_config['part_configs'] = part_configs if not 'parse' in section_config: section_config['parse'] = defaults.get( "parse", '(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)') if not 'serialize' in section_config: section_config['serialize'] = defaults.get( 'serialize', [str('{major}.{minor}.{patch}')]) if not 'search' in section_config: section_config['search'] = defaults.get( "search", '{current_version}') if not 'replace' in section_config: section_config['replace'] = defaults.get( "replace", '{new_version}') files.append( ConfiguredFile(filename, VersionConfig(**section_config))) else: message = "Could not read config file at {}".format(config_file) if explicit_config: raise argparse.ArgumentTypeError(message) else: logger.info(message) parser2 = argparse.ArgumentParser(prog='bumpversion', add_help=False, parents=[parser1]) parser2.set_defaults(**defaults) parser2.add_argument('--current-version', metavar='VERSION', help='Version that needs to be updated', required=False) parser2.add_argument('--parse', metavar='REGEX', help='Regex parsing the version string', default=defaults.get( "parse", '(?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)')) parser2.add_argument('--serialize', metavar='FORMAT', action=DiscardDefaultIfSpecifiedAppendAction, help='How to format what is parsed back to a version', default=defaults.get( "serialize", [str('{major}.{minor}.{patch}')])) parser2.add_argument('--search', metavar='SEARCH', help='Template for complete string to search', default=defaults.get("search", '{current_version}')) parser2.add_argument('--replace', metavar='REPLACE', help='Template for complete string to replace', default=defaults.get("replace", '{new_version}')) known_args, remaining_argv = parser2.parse_known_args(args) defaults.update(vars(known_args)) assert type(known_args.serialize) == list context = dict( list(time_context.items()) + list(prefixed_environ().items()) + list(vcs_info.items())) try: vc = VersionConfig( parse=known_args.parse, serialize=known_args.serialize, search=known_args.search, replace=known_args.replace, part_configs=part_configs, ) except sre_constants.error as e: sys.exit(1) current_version = vc.parse( known_args.current_version) if known_args.current_version else None new_version = None if not 'new_version' in defaults and known_args.current_version: try: if current_version and len(positionals) > 0: logger.info("Attempting to increment part '{}'".format( positionals[0])) new_version = current_version.bump(positionals[0], vc.order()) logger.info("Values are now: " + keyvaluestring(new_version._values)) defaults['new_version'] = vc.serialize(new_version, context) except MissingValueForSerializationException as e: logger.info("Opportunistic finding of new_version failed: " + e.message) except IncompleteVersionRepresenationException as e: logger.info("Opportunistic finding of new_version failed: " + e.message) except KeyError as e: logger.info("Opportunistic finding of new_version failed") parser3 = argparse.ArgumentParser( prog='bumpversion', description=DESCRIPTION, formatter_class=argparse.ArgumentDefaultsHelpFormatter, conflict_handler='resolve', parents=[parser2], ) parser3.set_defaults(**defaults) parser3.add_argument('--current-version', metavar='VERSION', help='Version that needs to be updated', required=not 'current_version' in defaults) parser3.add_argument('--dry-run', '-n', action='store_true', default=False, help="Don't write any files, just pretend.") parser3.add_argument('--new-version', metavar='VERSION', help='New version that should be in the files', required=not 'new_version' in defaults) commitgroup = parser3.add_mutually_exclusive_group() commitgroup.add_argument('--commit', action='store_true', dest="commit", help='Commit to version control', default=defaults.get("commit", False)) commitgroup.add_argument('--no-commit', action='store_false', dest="commit", help='Do not commit to version control', default=argparse.SUPPRESS) taggroup = parser3.add_mutually_exclusive_group() taggroup.add_argument('--tag', action='store_true', dest="tag", default=defaults.get("tag", False), help='Create a tag in version control') taggroup.add_argument('--no-tag', action='store_false', dest="tag", help='Do not create a tag in version control', default=argparse.SUPPRESS) parser3.add_argument('--tag-name', metavar='TAG_NAME', help='Tag name (only works with --tag)', default=defaults.get('tag_name', 'v{new_version}')) parser3.add_argument( '--message', '-m', metavar='COMMIT_MSG', help='Commit message', default=defaults.get( 'message', 'Bump version: {current_version} → {new_version}')) file_names = [] if 'files' in defaults: assert defaults['files'] != None file_names = defaults['files'].split(' ') parser3.add_argument('part', help='Part of the version to be bumped.') parser3.add_argument('files', metavar='file', nargs='*', help='Files to change', default=file_names) args = parser3.parse_args(remaining_argv + positionals) if args.dry_run: logger.info("Dry run active, won't touch any files.") if not new_version: new_version = vc.parse(args.new_version) logger.info("New version will be '{}'".format(args.new_version)) file_names = file_names or positionals[1:] for file_name in file_names: files.append(ConfiguredFile(file_name, vc)) for vcs in VCS: if vcs.is_usable(): try: vcs.assert_nondirty() except WorkingDirectoryIsDirtyException as e: if not defaults['allow_dirty']: logger.warn( "{}\n\nUse --allow-dirty to override this if you know what you're doing." .format(e.message)) raise break else: vcs = None # make sure files exist and contain version string logger.info("Asserting files {} contain the version string:".format( ", ".join([str(f) for f in files]))) for f in files: f.should_contain_version(current_version, context) # change version string in files for f in files: f.replace(current_version, new_version, context, args.dry_run) commit_files = [f.path for f in files] config.set('bumpversion', 'new_version', args.new_version) for key, value in config.items('bumpversion'): logger_list.info("{}={}".format(key, value)) config.remove_option('bumpversion', 'new_version') config.set('bumpversion', 'current_version', args.new_version) new_config = StringIO() try: write_to_config_file = (not args.dry_run) and config_file_exists logger.info("{} to config file {}:".format( "Would write" if not write_to_config_file else "Writing", config_file, )) config.write(new_config) logger.info(new_config.getvalue()) if write_to_config_file: with io.open(config_file, 'wb') as f: f.write(new_config.getvalue().encode('utf-8')) except UnicodeEncodeError: warnings.warn( "Unable to write UTF-8 to config file, because of an old configparser version. " "Update with `pip install --upgrade configparser`.") if config_file_exists: commit_files.append(config_file) if not vcs: return assert vcs.is_usable(), "Did find '{}' unusable, unable to commit.".format( vcs.__name__) do_commit = (not args.dry_run) and args.commit do_tag = (not args.dry_run) and args.tag logger.info("{} {} commit".format( "Would prepare" if not do_commit else "Preparing", vcs.__name__, )) for path in commit_files: logger.info("{} changes in file '{}' to {}".format( "Would add" if not do_commit else "Adding", path, vcs.__name__, )) if do_commit: vcs.add_path(path) vcs_context = { "current_version": args.current_version, "new_version": args.new_version, } vcs_context.update(time_context) vcs_context.update(prefixed_environ()) commit_message = args.message.format(**vcs_context) logger.info("{} to {} with message '{}'".format( "Would commit" if not do_commit else "Committing", vcs.__name__, commit_message, )) if do_commit: vcs.commit(message=commit_message) tag_name = args.tag_name.format(**vcs_context) logger.info("{} '{}' in {}".format( "Would tag" if not do_tag else "Tagging", tag_name, vcs.__name__)) if do_tag: vcs.tag(tag_name)
from __future__ import absolute_import, division, print_function, unicode_literals from datetime import datetime import os from configparser import RawConfigParser AI_DIR = os.path.dirname(os.path.abspath(__file__)) BASE_DIR = os.path.dirname(AI_DIR) # config_setting = RawConfigParser() # config_setting.read(BASE_DIR+'/setting.ini') config_version = RawConfigParser() config_version.read(BASE_DIR+'/version.cfg') version = '{}.{}'.format(config_version.get('MAIN', 'V'), config_version.get('MAIN', 'SUR')) def print_spend_time(_st_time): _ed_time = datetime.now() # _spend_seconds = (_ed_time - _st_time).total_seconds() # _left_seconds = int(_spend_seconds % 60) _spend_minutes = int(_spend_seconds // 60) _left_minutes = int(_spend_minutes % 60) _left_hours = int(_spend_minutes // 60) print('==== spend time: {:d} h: {:d} m: {:d} s'.format(_left_hours, _left_minutes, _left_seconds)) return _spend_seconds def check_path(path): if not os.path.exists(path): os.makedirs(path) return path
def import_eds(source, node_id): eds = RawConfigParser() if hasattr(source, "read"): fp = source else: fp = open(source) try: # Python 3 eds.read_file(fp) except AttributeError: # Python 2 eds.readfp(fp) fp.close() od = objectdictionary.ObjectDictionary() if eds.has_section("DeviceComissioning"): od.bitrate = int(eds.get("DeviceComissioning", "Baudrate")) * 1000 od.node_id = int(eds.get("DeviceComissioning", "NodeID")) for section in eds.sections(): # Match indexes match = re.match(r"^[0-9A-Fa-f]{4}$", section) if match is not None: index = int(section, 16) name = eds.get(section, "ParameterName") try: object_type = int(eds.get(section, "ObjectType"), 0) except NoOptionError: # DS306 4.6.3.2 object description # If the keyword ObjectType is missing, this is regarded as # "ObjectType=0x7" (=VAR). object_type = VAR if object_type in (VAR, DOMAIN): var = build_variable(eds, section, node_id, index) od.add_object(var) elif object_type == ARR and eds.has_option(section, "CompactSubObj"): arr = objectdictionary.Array(name, index) last_subindex = objectdictionary.Variable( "Number of entries", index, 0) last_subindex.data_type = objectdictionary.UNSIGNED8 arr.add_member(last_subindex) arr.add_member(build_variable(eds, section, node_id, index, 1)) od.add_object(arr) elif object_type == ARR: arr = objectdictionary.Array(name, index) od.add_object(arr) elif object_type == RECORD: record = objectdictionary.Record(name, index) od.add_object(record) continue # Match subindexes match = re.match(r"^([0-9A-Fa-f]{4})[S|s]ub([0-9A-Fa-f]+)$", section) if match is not None: index = int(match.group(1), 16) subindex = int(match.group(2), 16) entry = od[index] if isinstance(entry, (objectdictionary.Record, objectdictionary.Array)): var = build_variable(eds, section, node_id, index, subindex) entry.add_member(var) # Match [index]Name match = re.match(r"^([0-9A-Fa-f]{4})Name", section) if match is not None: index = int(match.group(1), 16) num_of_entries = int(eds.get(section, "NrOfEntries")) entry = od[index] # For CompactSubObj index 1 is were we find the variable src_var = od[index][1] for subindex in range(1, num_of_entries + 1): var = copy_variable(eds, section, subindex, src_var) if var is not None: entry.add_member(var) return od
def test_get_config_value_None_bool(self): config = RawConfigParser() config["section"] = {"key": None} val = get_config_value(config, "section", "key", dtype=bool) self.assertEqual(val, False)