def run_main(): global configfile global maintenance_job, legacy_sync_job global main_xmlrpc_handler, stats_resource, web_root, web_static parser = argparse.ArgumentParser(description="DenyHosts sync server") parser.add_argument("-c", "--config", default="/etc/denyhosts-server.conf", help="Configuration file") parser.add_argument("--recreate-database", action='store_true', help="Wipe and recreate the database") parser.add_argument("--evolve-database", action='store_true', help="Evolve the database to the latest schema version") parser.add_argument("--purge-legacy-addresses", action='store_true', help="Purge all hosts downloaded from the legacy server. DO NOT USE WHEN DENYHOSTS-SERVER IS RUNNING!") parser.add_argument("--purge-reported-addresses", action='store_true', help="Purge all hosts that have been reported by clients. DO NOT USE WHEN DENYHOSTS-SERVER IS RUNNING!") parser.add_argument("--purge-ip", action='store', help="Purge ip address from both legacy and reported host lists. DO NOT USE WHEN DENYHOSTS-SERVER IS RUNNING!") parser.add_argument("--check-peers", action="store_true", help="Check if all peers are responsive, and if they agree about the peer list") parser.add_argument("--bootstrap-from-peer", action="store", metavar="PEER_URL", help="First wipe database and then bootstrap database from peer. DO NOT USE WHEN DENYHOSTS-SERVER IS RUNNING!") parser.add_argument("-f", "--force", action='store_true', help="Do not ask for confirmation, execute action immediately") args = parser.parse_args() configfile = args.config try: config.read_config(args.config) except ConfigParser.NoSectionError, e: print("Error in reading the configuration file from \"{}\": {}.".format(args.config, e)) print("Please review the configuration file. Look at the supplied denyhosts-server.conf.example for more information.") sys.exit()
def do_set(self, args): """ Set config for cloudmonkey. For example, options can be: url, auth, log_file, history_file You may also edit your ~/.cloudmonkey_config instead of using set. Example: set url http://localhost:8080/client/api set prompt 🐵 cloudmonkey> set log_file /var/log/cloudmonkey.log """ args = args.strip().partition(" ") key, value = (args[0].strip(), args[2].strip()) if not key: return if not value: print "Blank value of %s is not allowed" % key return self.prompt = self.get_prompt() setattr(self, key, value) if key in ['host', 'port', 'path', 'protocol']: key = 'url' self.url = "%s://%s:%s%s" % (self.protocol, self.host, self.port, self.path) print "This option has been deprecated, please set 'url' instead" print "This server url will be used:", self.url write_config(self.get_attr, self.config_file) read_config(self.get_attr, self.set_attr, self.config_file) self.init_credential_store() if key.strip() == 'profile': print "\nLoaded server profile '%s' with options:" % value for option in default_profile.keys(): print " %s = %s" % (option, self.get_attr(option)) print
def test(): import cairo surface = cairo.ImageSurface(cairo.FORMAT_ARGB32,100,100) context = cairo.Context(surface) config.read_config(path("/home/jose/.astronex")) zod = Zodiac(context) for z in zod.zod: print z.__dict__ print for z in zod.plan: print z.__dict__
def install_nifi(): logger.info('Attempting to install NiFi to the cluster') if not is_ambari_installed(): logger.error('Ambari must be installed to install NiFi as well.') raise EnvironmentError('You must install the demo on the same node as the Ambari server. Install Ambari here or move to another node with Ambari installed before continuing') if not is_hdp_select_installed(): installed = install_hdp_select() if not installed: logger.error('hdp-select must be installed to install NiFi') raise EnvironmentError('hdp-select could not be installed. Please install it manually and then re-run the setup.') conf = config.read_config('service-installer.conf') cmds = json.loads(conf['NIFI']['install-commands']) sh = Shell() logger.info('Getting HDP Version') version = sh.run(cmds[0]) logger.info('HDP Version: ' + version[0]) fixed_copy = cmds[2].replace('$VERSION', str(version[0])).replace('\n', '') fixed_remove = cmds[1].replace('$VERSION', str(version[0])).replace('\n', '') logger.info('NiFi Clean Command: ' + fixed_copy) logger.info('NiFi Copy Command: ' + fixed_remove) remove = sh.run(fixed_remove) copy = sh.run(fixed_copy) logger.info('Attempting to restart Ambari...') restart = sh.run(cmds[3]) print("Please open the Ambari Interface and manually deploy the NiFi Service.") raw_input("Press enter twice to continue...") raw_input("Press enter once to continue...") # We've copied the necessary files. Once that completes we need to add it to Ambari logger.info('Waiting for user to install service in Ambari to continue') print('Checking to make sure service is installed') ambari = config.read_config('global.conf')['AMBARI'] installed = check_ambari_service_installed('NIFI', ambari) logger.info('NiFi installed successfully') cont = '' if not installed: print('Unable to contact Ambari Server. Unsure whether or not Zeppelin was installed') while not (cont == 'y' or cont == 'n'): cont = raw_input('Continue attempt to set up NiFi for demo?(y/n)') if not (cont == 'y' or cont == 'n'): print('Please enter "y" or "n"') else: cont = 'y' if cont == 'n': return False elif cont == 'y': return True
def general(sqlsession): active = 'settings' devices = sqlsession.query(models.Device).all() return render_template( 'settings_general.html', active=active, category='general', flash_device=config.read_config('flash_device'), default_door_device=config.read_config('default_door_device'), devices=devices, semester_end_dates=config.read_config('semester_end_dates') )
def slice(jobinfo): # this will be replaced with proper error checking code later assert(jobinfo.status == 'slicing') conf = config.read_config() # read and store config information # this allows us to have different version of slicer stored in different places slice_command = conf['Slicer']['Path'] slice_ini = conf['Slicer']['Config'] # get the last file worked on (validated file) infile = jobinfo.previous_file() # CK - Modified to work on my linux VM - passes # Infile OK as is, must do some work for correct outfile # Remove original file extension and add .gcode extension # AW - fixed to work with file paths that contain periods # put together the input and output filenames root, ext = os.path.splitext(infile) outfile = root+'.gcode' # construct the actual command line call # CK - Removed quotes around infile and outfile names command = '{0} {1} --output {2} --load {3}'.format(slice_command, infile, outfile, slice_ini) # execute the command line call. # output contains anything written to stdout by the program # CK - Uncommented line output = os.popen(command).read() # this is a debugging message. remove on release slice_logger.log("$ "+command) # store the newly created file into the printjob jobinfo['slicing'] = outfile
def authorize_token(user): config = read_config() if config.get('GitHub', 'token'): print("The token already exists.") sys.exit() password = getpass('Password for {0}: '.format(user)) note = 'OCA (odoo community association) Maintainers Tools' note_url = 'https://github.com/OCA/maintainers-tools' scopes = ['repo', 'read:org', 'write:org', 'admin:org'] try: auth = github3.authorize(user, password, scopes, note, note_url) except github3.GitHubError as err: if err.code == 422: for error in err.errors: if error['code'] == 'already_exists': msg = ("The 'OCA (odoo community association) Maintainers " "Tools' token already exists. You will find it at " "https://github.com/settings/applications and can " "revoke it or set the token manually in the " "configuration file.") sys.exit(msg) raise config.set("GitHub", "token", auth.token) write_config(config) print("Token stored in configuration file")
def head(self): if not self.config: self.configure(config.read_config()) if not self.blockchain or 'HEAD' not in self.blockchain: self._initialize_blockchain() ptr = self.blockchain.get('HEAD') return blocks.get_block(self.blockchain, ptr)
def print_tabulate(res, offering_type="compute", noheader=False, short=False): config = read_config() c = conn(config) tbl = [] for i in res: if short: tbl.append([i.name]) else: if offering_type == "compute": tbl.append([ i.name, i.displaytext, i.storagetype, ]) elif offering_type == "network": tbl.append([ i.name, i.displaytext, "%s Mbit/s"% i.networkrate, ]) tbl = sorted(tbl, key=operator.itemgetter(0)) if (noheader or short): print tabulate(tbl, tablefmt="plain") else: if offering_type == 'compute': tbl.insert(0, ['name', 'description', 'storagetype']) else: tbl.insert(0, ['name', 'description', 'external networkrate']) print tabulate(tbl, headers="firstrow")
def post_notebook(notebook_path, session_cookie=''): '''Add a single notebook to a Zeppelin installation via REST API Must have a file called ``global.conf`` inside of the configuration directory. Inside that configuration file we use protocol://server:port to connect to a Zeppelin instance and use the API Args: notebook_path (str): Full file path to the Zeppelin note session_cookie (str, optional): The cookie used after authentication in order to make authorized API calls. Required on the 2.5 TP Sandbox Returns: bool: True if the upload was successful (received 201 created), or False otherwise. ''' conf = config.read_config('global.conf')['ZEPPELIN'] client = CurlClient(proto=conf['protocol'], server=conf['server'], port=int(conf['port'])) path = '/api/notebook' logger.info('Attempting to POST notebook at ' + client.proto + '://' + client.server + ':' + str(client.port)) opts = '-i -b \'' + session_cookie + '\' -H "Content-Type: application/json" -d @' + notebook_path output = client.make_request('POST', path, options=opts) if '201 created' in output[0].lower(): logger.info('Note posted successfully') return True else: logger.info('Note failed to be added. User will need to add manually') return False
def main(nzb_files, options): if not nzb_files: parser.print_help() return settings = config.read_config(options.config_file) if not settings: print '%s has no settings!' % options.config_file return if options.debug: settings['debug'] = options.debug print settings if options.pattern: settings['skip_regex'] = options.pattern settings['invert'] = True elif options.par2: settings['skip_regex'] = ['\.par2'] settings['invert'] = True nzbs = [nzb for nzb in helper.get_nzb_file(nzb_files)] for nzb in nzbs: nzb_name = os.path.split(nzb)[1] new_dir = helper.get_download_path(settings.get('download_dir'), nzb_name) settings['download_path'] = new_dir if not os.path.exists(new_dir): if settings.get('debug'): print 'made new dir %s ' % new_dir os.mkdir(new_dir) if settings.get('debug'): print settings['download_path'] download.start(nzb, settings)
def add_zeppelin_notebooks(): '''Add all Zeppelin notes to the current Zeppelin instalation Looks inside ``configuration/zeppelin/notes`` for any ``*.json`` files. Each JSON file will be attempted to be added to the current Zeppelin instance Returns: bool: True is every Note was added successfully. True is returned on an empty directory. False otherwise. ''' logger.info('Adding zeppelin notebooks to installation') all_success = True note_dir = config.get_conf_dir() + 'zeppelin/notes' # First need to authenticate to Zeppelin and retrieve JSESSIONID conf = config.read_config('global.conf')['ZEPPELIN'] session = get_zeppelin_session(conf['username'], conf['password']) for item in os.listdir(note_dir): logger.debug('Found item in zeppelin/notes: ' + item) item_path = note_dir + '/' + item if os.path.isfile(item_path) and str(item).endswith('.json'): logger.info('POSTing ' + item + ' to Zeppelin') result = post_notebook(item_path, session) if not result: logger.error('Not all notebooks were added to Zeppelin successfully') all_success = False return all_success
def print_tabulate(res, noheader=False, short=False): config = read_config() c = conn(config) tbl = [] for i in res: if not i.__dict__.has_key("associatednetworkname"): if i.__dict__.has_key("vpcid"): i.__setattr__("associatednetworkname", "%s (VPC)" % c.list_vpcs(id=i.vpcid)[0].name) if i.isstaticnat: vmname = i.virtualmachinename else: vmname = "" if i.issourcenat: snat = "Yes" else: snat = "No" if short: tbl.append([i.name]) else: tbl.append([i.ipaddress, i.zonename, i.associatednetworkname, vmname, snat, i.allocated]) tbl = sorted(tbl, key=operator.itemgetter(2)) if noheader or short: print tabulate(tbl, tablefmt="plain") else: tbl.insert(0, ["ip", "zone", "network", "staticnat", "sourcenat", "allocated"]) print tabulate(tbl, headers="firstrow")
def get_driver(): driver = config.read_config('driver') if driver == 'chrome': browser = webdriver.Chrome(config.read_config('chromedriver_path')) browser.maximize_window() browser.implicitly_wait(30) elif driver == 'android': desired_capabilities = {'aut': config.read_config('android_app_id')} return webdriver.Remote( desired_capabilities=desired_capabilities ) else: browser = webdriver.Firefox() browser.maximize_window() browser.implicitly_wait(30) return browser
def print_tabulate(res, noheader=False, short=False, t='list'): config = read_config() c = conn(config) tbl = [] for i in res: if t == 'list': if not i.__dict__.has_key('virtualmachinename'): i.__setattr__('virtualmachinename', 'None') if short: tbl.append([i.id]) else: tbl.append([ i.ipaddress, c.list_networks(id=i.networkid)[0].name, i.publicport, i.privateport, i.protocol, i.virtualmachinename, i.state, i.id, ]) tbl = sorted(tbl, key=operator.itemgetter(0)) if (noheader or short): print tabulate(tbl, tablefmt="plain") else: tbl.insert(0, ['ip', 'network', 'public port', 'private port', 'proto', 'vmname', 'state', 'uuid']) print tabulate(tbl, headers="firstrow")
def print_tabulate(res, noheader=False, short=False): config = read_config() c = conn(config) tbl = [] for i in res: if not i.__dict__.has_key('passwordenabled'): i.__setattr__('passwordenabled', 0) if not i.__dict__.has_key('created'): i.__setattr__('created', '') if i.passwordenabled == 1: passw = "Yes" else: passw = "No" if short: tbl.append(["%s/%s" % (i.account, i.name)]) else: tbl.append([ "%s/%s" % (i.account, i.name), i.zonename, i.ostypename, i.created, passw, ]) tbl = sorted(tbl, key=operator.itemgetter(0)) if (noheader or short): print tabulate(tbl, tablefmt="plain") else: tbl.insert(0, ['name', 'zone', 'ostype', 'created', 'passwordenabled']) print tabulate(tbl, headers="firstrow")
def get_zeppelin_session(username, password): '''Gets the JSESSIONID cookie by POSTing to /api/login This is required to retrieve the JSESSIONID on Zeppelin instances that have logins and user permissions. Used when POSTing notebooks as well. Returned in the form of ``'JSESSIONID=**`` Args: username (str): The username to login with password (str): The password to login with Returns: str: ``''`` If the request was unsuccessful, or the cookie wasn't present. Otherwise returns the cookie as a string. ''' conf = config.read_config('global.conf')['ZEPPELIN'] client = CurlClient(proto=conf['protocol'], server=conf['server'], port=int(conf['port'])) path = '/api/login' opts = '-i -d \'userName='******'&password='******'\'' res = client.make_request('POST', path, options=opts) lines = res[0].split('\n') for line in lines: if 'Set-Cookie:'.lower() in line.lower() and 'JSESSIONID' in line.upper(): cookie = line.split('Set-Cookie: ')[1] print 'COOKIE: ' + cookie return cookie.strip() return ''
def env(environ, content): "show the environment (debugging/tracing)" configpath = config.get_configpath() ctimef = config.get_modified_configpath_fs(configpath) sctimef = config.fmt_ts(ctimef) if ctimef else "None" ctimeu = config.get_modified_configpath_db(configpath) sctimeu = config.fmt_ts(ctimeu) if ctimeu else "never" content.append(u"""<table> <tr><td>%s</td><td>%s</td></tr> <tr><td>%s</td><td>%s</td></tr> <tr><td>%s</td><td>%s</td></tr> </table>""" % ("configpath", configpath, "modified", sctimef, "modified of read", sctimeu)) content.append(u"<br /><strong>contents of %s </strong><br />" % configpath) for entry in config.read_config(configpath): content.append(str(entry) + "<br />") content.append(u"<br /> <br /><strong>environ</strong>") # the most important ones first important = ("wsgi.multithread", "wsgi.multiprocess") for key in important: content.append(_envvar(environ, key)) # everything else for key in environ: if key not in important: content.append(_envvar(environ, key))
def main(): # read the config file. We'll need this later. # also caches the file for stages configrc = config.read_config() #load the logger plogger = logger.Logger('pipeline') if len(sys.argv) == 1: plogger.log("Error: no model given to pipeline") return infile = sys.argv[1] job = pipeline.PrintJob(infile, "*****@*****.**") job.status = 'converting' converter.convert(job) #replaced by slic3r automatic validation #job.status = 'validating' #validator.validate(job) job.status = 'slicing' slicer.slice(job) job.status = 'printing' printer.send_job(job)
def copy_inputs(config_file, InputsDir): config_dict = read_config(config_file) config = SafeConfigParser() config.optionxform = str config.read(config_file) new_config = os.path.join(InputsDir, os.path.split(config_file)[1]) # ---------------------------------------------------------------- # # copy the inputs for key, section in config_dict.iteritems(): if 'FILE_NAME' in section.keys(): new_file_name = os.path.join(InputsDir, os.path.split(section['FILE_NAME'])[1]) copyfile(section['FILE_NAME'], new_file_name) # update the config file for an easy restart config.set(key, 'FILE_NAME', os.path.join(InputsDir, os.path.split(section['FILE_NAME'])[1])) # update the config_dict with the new value config_dict[key]['FILE_NAME'] = new_file_name # ---------------------------------------------------------------- # # ---------------------------------------------------------------- # # write the new configuration file with open(new_config, 'w') as configfile: config.write(configfile) # ---------------------------------------------------------------- # return config_dict
def convert(jobinfo): # this should be replaced by error handling code later assert(jobinfo.status == 'converting') #read config file information conf = config.read_config() conv_command = conf['Converter']['Path'] # determine input and output filenames infile = jobinfo.previous_file() root, ext = os.path.splitext(infile) if ext not in conf['Mailfetch']['extensions']: conv_logger.error(jobinfo, 'File is in an unsupported format {0}'.format(ext)) outfile = None # don't bother converting if you already have an stl if ext != '.stl': outfile = root+'.stl' # run command command = '{0} {1} -c stl -o {2}.stl'.format(conv_command, infile, root) #output = os.popen(command).read() conv_logger.log("$ "+command) else: conv_logger.log("$ Converter: {0} is already an stl file".format(infile)) # if you did not convert, this is set to None # this will simply tell the pipeline that no file was created in this stage jobinfo['converting'] = outfile
def decrypt_config(path="./config"): """ Decrypt all json.encrypt files to .json files """ title_screen() password = utils.get_master_password() print "Processing folder `%s`" % path print(green("Decrypting...")) for root, dirs, files in os.walk(path): for file in files: if file.endswith(".encrypt"): encrypt_file_path = os.path.join(root, file) with open(encrypt_file_path) as ciphertext: file_path = encrypt_file_path[:-8] try: config_data = config.read_config(file_path) config.write_config(file_path, config_data) print(green("File `%s` decrypted." % encrypt_file_path)) os.remove(encrypt_file_path) except: print(red("Cannot decrypt file... `%s` skipping..." % encrypt_file_path))
def __init__(self, logfilepath = "%s/var/log/certmaster/certmaster.log" % conf_dir): config_file = '%s/etc/certmaster/minion.conf' % conf_dir self.config = read_config(config_file, CMConfig) self.loglevel = logging._levelNames[self.config.log_level] self._setup_logging() if self._no_handlers: self._setup_handlers(logfilepath=logfilepath)
def main(): config = read_config() c = conn(config) args = docopt(__doc__, version='%s %s' % (__cli_name__, __version__)) if args['list']: res = c.list_sshkeypairs() if args['generate']: try: res = c.create_sshkeypair(name=args['NAME']) print res.privatekey sys.exit(0) except molnctrl.csexceptions.ResponseError as e: print "Unable to create ssh key: %s" % e[1] sys.exit(1) if args['delete']: try: res = c.delete_sshkeypair(name=args['NAME']) print "deleted SSH key: %s" % args['NAME'] sys.exit(0) except Exception as e: print "Unable to delete SSK keypair: %s" % e[1] sys.exit(1) if res: print_tabulate(res, noheader=args['--noheader'], short=args['--short']) else: print "Unable to execute command" sys.exit(1)
def print_tabulate(res, noheader=False, short=False): config = read_config() c = conn(config) tbl = [] for i in res: if not i.__dict__.has_key('networkofferingname'): i.__setattr__('networkofferingname', 'VPC') if short: tbl.append([i.name]) else: tbl.append([ i.name, i.zonename, i.networkofferingname, i.cidr, i.networkdomain, get_snat(c, i.id), has_vpn(c, i.id), i.state ]) tbl = sorted(tbl, key=operator.itemgetter(0)) if (noheader or short): print tabulate(tbl, tablefmt="plain") else: tbl.insert(0, ['name', 'zone', 'offering', 'cidr', 'domain', 'snat', 'vpn', 'state']) print tabulate(tbl, headers="firstrow")
def print_tabulate(res, noheader=False, short=False, t="vpn"): config = read_config() c = conn(config) tbl = [] for i in res: if t == 'vpn': if short: tbl.append([i.publicip]) else: tbl.append([ get_network_for_ip(c, i.publicipid), i.publicip, i.presharedkey, i.state, ]) if t == 'vpnusers': if short: tbl.append([i.username]) else: tbl.append([ i.username, i.state, ]) tbl = sorted(tbl, key=operator.itemgetter(0)) if (noheader or short): print tabulate(tbl, tablefmt="plain") else: if t == 'vpn': tbl.insert(0, ['network', 'endpoint', 'preshared key', 'state']) elif t == 'vpnuser': tbl.insert(0, ['username', 'state']) print tabulate(tbl, headers="firstrow")
def before_feature(context, scenario): driver = config.read_config('driver') if driver == 'android': pass else: context.execute_steps(u''' given login to eTools ''')
def after_feature(context, scenario): driver = config.read_config('driver') if driver == 'android': pass else: context.execute_steps(u''' given logout from eTools ''')
def before_all(context): setup_debug_on_error(context.config.userdata) context.browser = get_driver() context.util = Utils(context.browser) context.base_url = config.read_config('baseurl') context.verificationErrors = [] context.accept_next_alert = True
def update(self, config_file = None): self._apps = dict() self._windows = [] # Load running apps config.read_config(config_file) apps = elements.get_accessible_applications(config.IGNORED_BUNDLES) for app in apps: self._apps[app.title] = app app._element.set_callback(_accessibility_notifications_callback) app._element.watch('AXWindowMiniaturized', 'AXWindowCreated') for win in app._windows: self._add_window(win) logging.info('The window manager is now aware of: %s', ', '.join(self._apps.keys())) self._layout = config.LAYOUT
#%% import os import sys # sys.path.append(os.getcwd().replace(f"\classification", "")) from classification.model_evaluation import print_cross_val_scores print(f"Working directory: {os.getcwd()}") for path in sys.path: print(path) #%% import config from dataset import dataset_loader dataset = dataset_loader.load_dataset(config.read_config('')) #%% Add targets to features as a new column # dataset['features']['targets'] = dataset['targets'] #%% -- print(dataset['features'].info()) print(dataset['targets'].value_counts()) #%% Create histogram of the features to see their shame -- import matplotlib.pyplot as plt dataset['features'].hist(bins=50, figsize=(10, 7)) plt.show() #%% dataset['targets'].hist(bins=50, figsize=(5, 3)) plt.show() #%% Looking for correlations
from model_rvrd_kepler import lnlike, lnprior, preprocess import config import numpy as np nplanets = 0 modelpath = 'configfiles/hd40307_dace.py' parnames, datadict, priordict, fixedpardict = config.read_config( modelpath, nplanets, False) theta = [31334.88, 0.81, -0.16, -0.04] print(parnames) likelihood = lnlike(theta, parnames, fixedpardict, datadict, {}) print(f'Log(L) = {-likelihood}')
def do_POST(self): if (self.path == "/get") or (self.path == "/train"): self.send_response(200) else: self.send_response(404) self.end_headers() return form = cgi.FieldStorage(fp=self.rfile, headers=self.headers, environ={ "REQUEST_METHOD": "POST", "CONTENT_TYPE": self.headers["Content-Type"] }) field_item = form["file"] if (field_item == None): self.send_response(403) self.end_headers() return filename = field_item.filename fileValue = field_item.value fileSize = len(fileValue) print("[UPLOAD]", "file: ", filename, ", size:", len(fileValue), "bytes") if (len(fileValue) > 20000000) or not ( filename.endswith(".jpg") or filename.endswith(".png") or filename.endswith(".jpeg") or filename.endswith(".gif")): self.send_response(403) self.end_headers() return else: self.send_header("Access-Control-Allow-Origin", "*") self.send_header("Content-type", "application/json") self.end_headers() if self.path == "/get": with open("upload/" + filename, "wb") as f: f.write(fileValue) crop.ssd_process(img_path="upload/" + filename, crop_path="crop", show=False) os.remove("upload/" + filename) files = glob.glob(crop_path + "/*") pred_result = [] pred_probabilities = [] for index, f in enumerate(files): img = image.load_img(f, target_size=(256, 256)) image_tensor = image.img_to_array(img) image_tensor = np.expand_dims(image_tensor, axis=0) image_tensor /= 255. model = load_model( "model/trash.h5", custom_objects={"leaky_relu": leaky_relu}) pred = model.predict(image_tensor) pred_class = np.argmax(pred, axis=1) pred_result.append(pred) pred_probabilities.append(pred) print("[PREDICT] Crop name:", f) print("[PREDICT] Raw prediction data:", pred) print("[PREDICT] Raw prediction class data: ", pred_class) response = { "status": "success", "result": pred_result, "probabilities": pred_probabilities } elif self.path == "/train": with open( "train/" + labels_index[int(form["type"].value)] + "/" + filename, "wb") as f: f.write(fileValue) response = {"status": "success"} if int(time.time()) - conf.read_config()["last"] > 86400: train_path = list(pathlib.Path("./train").glob("*/*")) train_count = len(train_path) if train_count > int(conf.read_config()["in"]): conf.write_config("last", int(time.time())) conf.write_config("in", weight.get_new_n(train_count)) train.do_train() for item in train_path: os.remove(item) responseBody = json.dumps(response) self.wfile.write(responseBody.encode("utf-8")) print()
def run_main(): global configfile global maintenance_job, legacy_sync_job global main_xmlrpc_handler, stats_resource, web_root, web_static parser = argparse.ArgumentParser(description="DenyHosts sync server") parser.add_argument("-c", "--config", default="/etc/denyhosts-server.conf", help="Configuration file") parser.add_argument("--recreate-database", action='store_true', help="Wipe and recreate the database") parser.add_argument( "--evolve-database", action='store_true', help="Evolve the database to the latest schema version") parser.add_argument( "--purge-legacy-addresses", action='store_true', help= "Purge all hosts downloaded from the legacy server. DO NOT USE WHEN DENYHOSTS-SERVER IS RUNNING!" ) parser.add_argument( "--purge-reported-addresses", action='store_true', help= "Purge all hosts that have been reported by clients. DO NOT USE WHEN DENYHOSTS-SERVER IS RUNNING!" ) parser.add_argument( "--purge-ip", action='store', help= "Purge ip address from both legacy and reported host lists. DO NOT USE WHEN DENYHOSTS-SERVER IS RUNNING!" ) parser.add_argument( "--check-peers", action="store_true", help= "Check if all peers are responsive, and if they agree about the peer list" ) parser.add_argument( "--bootstrap-from-peer", action="store", metavar="PEER_URL", help= "First wipe database and then bootstrap database from peer. DO NOT USE WHEN DENYHOSTS-SERVER IS RUNNING!" ) parser.add_argument( "-f", "--force", action='store_true', help="Do not ask for confirmation, execute action immediately") args = parser.parse_args() configfile = args.config try: config.read_config(args.config) except ConfigParser.NoSectionError, e: print( "Error in reading the configuration file from \"{}\": {}.".format( args.config, e)) print( "Please review the configuration file. Look at the supplied denyhosts-server.conf.example for more information." ) sys.exit()
filepath = os.path.dirname(__file__) modelpath = os.path.join( filepath, 'models/lognormal_prior/nathan_newmodel{}.py'.format(model)) # Generate dictionaries datafile = args_params.dfile data_path = os.path.join(filepath, 'data/data_2gen/') if args_params.cluster: data_path = '/home/spectro/nunger/codigo/src/nathan/data/data_2gen/' data_files = subprocess.check_output('ls {}'.format(data_path), shell=True).decode('utf-8').split('\n') data_files.remove('') # Remove last empty item assert datafile in range(1, 201), "Incorrect datafile has to be one of {}".format( range(1, 101)) parnames, datadict, priordict, fixedpardict = config.read_config( modelpath, data_path, datafile) if rank == 0: print('Datafile: {}'.format(data_files[datafile - 1])) print('\n Parameter names and order:') print(parnames) # Number of dimensions is the number of parameters in the model nDims = len(parnames) nDerived = 0 def logLikelihood(theta): """ Log Likelihood for the planet model. """ result = lnlike(theta, parnames, fixedpardict, datadict)
for run in runs: dates.append(run[-9:]) dates.sort() for run in runs: if dates[-1] in run: prev_run = run print('Corrida a resumir: {}'.format(prev_run)) # Extract the number of planets analyzed in previous run nplanets = int(prev_run.split('_')[1][0]) # Assign modelpath filepath = os.path.dirname(__file__) modelpath = os.path.join(filepath, 'configfiles/hd40307_model_newdata.py') # Generate dictionaries parnames, datadict, priordict, fixedpardict = config.read_config( modelpath, nplanets, args_params.narrow) # Filepath for the polychord data and storing the samples timecode = time.strftime("%m%d_%H%M") folder_path = 'hd40307_{}a_'.format(nplanets) + timecode if args_params.narrow: folder_path = 'hd40307_{}b_'.format(nplanets) + timecode if not args_params.save: # Save the samples in a dump folder if the data shouldn't be saved # This overwrites any data saved before of the same model folder_path = 'dump' base_dir = os.path.join(dirname, folder_path) if rank == 0: print('\n Parameter names and order:')
from flask import Flask, request from flask_restful import Resource, Api import config from scrape import ListApartment, SearchApartment import os app = Flask(__name__) api = Api(app) app_config = config.read_config() api.add_resource(SearchApartment, '/api/apartments/') api.add_resource(ListApartment, '/api/apartments/<int:zipcode>') app.run(port=app_config['application_port'], host='0.0.0.0', debug=True)
def __init__(self): config = read_config() self.__pg_connect_str = config["PostgreSQLConnectString"] self.__max_query_time = int(config["MaxQueryTimeSeconds"]) * 1000
def main(): location = get_user_input() configured_data = read_config() create_folders_if_necessary(list(configured_data.keys()), location) mapping_dictionary = get_mapping_dictionary(read_config()) move_files(mapping_dictionary, location)
# You should have received a copy of the GNU General # Public License along with Navidoc; if not, write to the Free # Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, # MA 02111-1307 USA # #$Id: rst2any.py,v 1.34 2003/07/22 11:03:50 humppake Exp $ # # Written by Asko Soukka # __docformat__ = 'reStructuredText' import config config.read_config(config, config.navidoc_conf) import sys, os, getopt import docutils.core import navidoc.directives import navidoc.writers import navidoc.languages import navidoc.transforms import navidoc.modules from navidoc.util.path import * # Import parser "plugins" from 'navidoc/link' and 'navidoc/mp' config.mp_includes = listdir('navidoc/mp',['mp'],dirs=0) dirlist = listdir('navidoc/mp',['py'],dirs=0)
def main(): """Top-level function.""" parser = argparse.ArgumentParser(description=__doc__.strip()) parser.add_argument('ledger', help="Beancount ledger file") parser.add_argument('config', action='store', help='Configuration for accounts and reports.') parser.add_argument('output', help="Output directory to write all output files to.") parser.add_argument( 'filter_reports', nargs='*', help="Optional names of specific subset of reports to analyze.") parser.add_argument('-v', '--verbose', action='store_true', help='Verbose mode') parser.add_argument('-d', '--days-price-threshold', action='store', type=int, default=5, help="The number of days to tolerate price latency.") parser.add_argument('-e', '--end-date', action='store', type=datetime.date.fromisoformat, help="The end date to compute returns up to.") parser.add_argument('--pdf', '--pdfs', action='store_true', help="Render as PDFs. Default is HTML directories.") parser.add_argument('-j', '--parallel', action='store_true', help="Run report generation concurrently.") parser.add_argument( '-E', '--check-explicit-flows', action='store_true', help=("Enables comparison of the general categorization method " "with the explicit one with specialized explicit handlers " "per signature.")) args = parser.parse_args() if args.verbose: logging.basicConfig(level=logging.DEBUG, format='%(levelname)-8s: %(message)s') logging.getLogger('matplotlib.font_manager').disabled = True # Figure out end date. end_date = args.end_date or datetime.date.today() # Load the example file. logging.info("Reading ledger: %s", args.ledger) entries, _, options_map = loader.load_file(args.ledger) accounts = getters.get_accounts(entries) dcontext = options_map['dcontext'] # Load, filter and expand the configuration. config = configlib.read_config(args.config, args.filter_reports, accounts) os.makedirs(args.output, exist_ok=True) with open(path.join(args.output, "config.pbtxt"), "w") as efile: print(config, file=efile) # Extract data from the ledger. account_data_map = investments.extract( entries, dcontext, config, end_date, args.check_explicit_flows, path.join(args.output, "investments")) # Generate output reports. output_reports = path.join(args.output, "reports") pricer = reports.generate_reports(account_data_map, config, prices.build_price_map(entries), end_date, output_reports, args.parallel, args.pdf) # Generate price reports. output_prices = path.join(args.output, "prices") reports.generate_price_pages(account_data_map, prices.build_price_map(entries), output_prices) # Output required price directives (to be filled in the source ledger by # fetching prices). reports.write_price_directives( path.join(output_prices, "prices.beancount"), pricer, args.days_price_threshold)
def connect_to_db(): db_conn_params = read_config(filename=POSTGRESQL_CONFIG_FILE_NAME, section="postgresql") conn = psycopg2.connect(**db_conn_params) conn.autocommit = False return conn
def main(config_file, num_opt_eval): conf = read_config(config_file) train_df, test_df = get_train_test(conf) feats = [ f for f in train_df.columns if f not in ( ['TARGET', 'SK_ID_CURR', 'SK_ID_BUREAU', 'SK_ID_PREV', 'index']) ] print(f"use {len(feats)} features.") parameter_space = { "learning_rate": hp.quniform("learning_rate", 0.01, 0.02, 0.001), "max_bin": hp.quniform("max_bin", 5, 500, 5), "num_leaves": hp.quniform("num_leaves", 16, 128, 1), "min_child_samples": hp.quniform("min_child_samples", 1, 300, 1), "colsample_bytree": hp.quniform("colsample_bytree", 0.01, 0.2, 0.01), "subsample": hp.quniform("subsample", 0.01, 0.8, 0.01), "min_gain_to_split": hp.quniform("min_gain_to_split", 0.0, 5.0, 0.1), "reg_alpha": hp.quniform("reg_alpha", 0, 100, 0.1), "reg_lambda": hp.quniform("reg_lambda", 0, 100, 0.1) } def objective(params): conf.model = { **conf.model, "clf_params": { "learning_rate": float(params["learning_rate"]), "max_bin": int(params["max_bin"]), "num_leaves": int(params["num_leaves"]), "min_child_samples": int(params["min_child_samples"]), "colsample_bytree": float(params["colsample_bytree"]), "subsample": float(params["subsample"]), "min_gain_to_split": float(params["min_gain_to_split"]), "reg_alpha": float(params["reg_alpha"]), "reg_lambda": float(params["reg_lambda"]), "boosting_type": "dart", "n_estimators": 10000, "max_depth": -1, "nthread": -1, "scale_pos_weight": 1, "is_unbalance": False, "silent": -1, "verbose": -1, "random_state": 0 } } pprint(conf.model.clf_params) model = LightGBM() score = model.train_and_predict_kfold(train_df, test_df, feats, 'TARGET', conf) return {'loss': -1.0 * score, 'status': STATUS_OK} print("====== optimize lgbm parameters ======") trials = hyperopt.Trials() best = fmin(objective, parameter_space, algo=tpe.suggest, max_evals=num_opt_eval, trials=trials, verbose=1) print("====== best estimate parameters ======") pprint(best) # for key, val in best.items(): # print(f" {key}: {val}") print("============= best score =============") best_score = -1.0 * trials.best_trial['result']['loss'] print(best_score) pickle.dump( trials.trials, open( os.path.join( conf.dataset.output_directory, f'{datetime.now().strftime("%m%d%H%M")}_{conf.config_file_name}_trials_score{best_score}.pkl' ), 'wb'))
def main(): parser = argparse.ArgumentParser(prog="sir") parser.add_argument("-d", "--debug", action="store_true") parser.add_argument("--sqltimings", action="store_true") subparsers = parser.add_subparsers() reindex_parser = subparsers.add_parser("reindex", help="Reindexes all or a single " "entity type") reindex_parser.set_defaults(func=reindex) reindex_parser.add_argument('--entity-type', action='append', help="Which entity types to index.", choices=SCHEMA.keys()) generate_trigger_parser = subparsers.add_parser("triggers", help="Generate triggers") generate_trigger_parser.set_defaults(func=generate_func) generate_trigger_parser.add_argument('-t', '--trigger-file', action="store", default="sql/CreateTriggers.sql", help="The filename to save the " "triggers into") generate_trigger_parser.add_argument('-f', '--function-file', action="store", default="sql/CreateFunctions.sql", help="The filename to save the " "functions into") generate_trigger_parser.add_argument('-bid', '--broker-id', action="store", default="1", help="ID of the AMQP broker row " "in the database.") generate_extension_parser = subparsers.add_parser( "extension", help="Generate extension") generate_extension_parser.set_defaults(func=generate_extension) generate_extension_parser.add_argument('-e', '--extension-file', action="store", default="sql/CreateExtension.sql", help="The filename to save the " "extension into") amqp_setup_parser = subparsers.add_parser("amqp_setup", help="Set up AMQP exchanges and " "queues") amqp_setup_parser.set_defaults(func=setup_rabbitmq) amqp_watch_parser = subparsers.add_parser("amqp_watch", help="Watch AMQP queues for " "changes") amqp_watch_parser.set_defaults(func=watch) args = parser.parse_args() if args.debug: logger.setLevel(logging.DEBUG) else: logger.setLevel(logging.INFO) loghandler = logging.StreamHandler() if args.debug: formatter = logging.Formatter(fmt="%(processName)s %(asctime)s " "%(levelname)s: %(message)s") else: formatter = logging.Formatter(fmt="%(asctime)s: %(message)s") loghandler.setFormatter(formatter) logger.addHandler(loghandler) mplogger = multiprocessing.get_logger() mplogger.setLevel(logging.ERROR) mplogger.addHandler(loghandler) if args.sqltimings: from sqlalchemy import event from sqlalchemy.engine import Engine import time sqltimelogger = logging.getLogger("sqltimer") sqltimelogger.setLevel(logging.DEBUG) sqltimelogger.addHandler(loghandler) @event.listens_for(Engine, "before_cursor_execute") def before_cursor_execute(conn, cursor, statement, parameters, context, executemany): conn.info.setdefault('query_start_time', []).append(time.time()) sqltimelogger.debug("Start Query: %s", statement) sqltimelogger.debug("With Parameters: %s", parameters) @event.listens_for(Engine, "after_cursor_execute") def after_cursor_execute(conn, cursor, statement, parameters, context, executemany): total = time.time() - conn.info['query_start_time'].pop(-1) sqltimelogger.debug("Query Complete!") sqltimelogger.debug("Total Time: %f", total) config.read_config() try: init_raven_client(config.CFG.get("sentry", "dsn")) except ConfigParser.Error as e: logger.info( "Skipping Raven client initialization. Configuration issue: %s", e) func = args.func args = vars(args) func(args)
body = request.urlopen(url=cfg["check_url"], timeout=10).read() return "synology" not in str(body).lower() except Exception as e: logging.warning("check url FAILED, error: %s", str(e)) return True def ask_exit(_sig_name): logging.warning('got signal {}: exit'.format(_sig_name)) loop.stop() if __name__ == '__main__': logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)-8s : %(message)s') logging.info('start...') read_config() check_config() cfg['record_id'] = get_record_id(cfg['domain'], cfg['sub_domain']) logging.info("get record_id: %s" % str(cfg['record_id'])) logging.info("watching ip for ddns: %s.%s" % (cfg['sub_domain'], cfg['domain'])) loop = asyncio.get_event_loop() for sig_name in ('SIGINT', 'SIGTERM'): try: loop.add_signal_handler(getattr(signal, sig_name), functools.partial(ask_exit, sig_name)) except NotImplementedError: pass # 使兼容 WINDOWS try: loop.run_until_complete(main()) except (KeyboardInterrupt, RuntimeError): logging.info('stop...')
config_file = options['config_path'] else: config_file += 'server.cfg' return config_file #-------------------------------------------------------------- # config affairs # look for default config name in lib/config.py args = sys.argv args = args[1:] options = command_line.parse_command_line(args) conf = config.read_config(get_config_filename(options)) override_config_with_command_line_options(conf, options) conf['GENERAL']['VERSION'] = '0.9.9.0.2' print 'NTLM authorization Proxy Server v%s' % conf['GENERAL']['VERSION'] print 'Copyright (C) 2001-2012 by Tony Heupel, Dmitry Rozmanov, and others.' config = config_affairs.arrange(conf) #-------------------------------------------------------------- # let's run it serv = server.AuthProxyServer(config) serv.run()
import cv2 as cv import numpy as np import os import os.path import config import logging from camera_setup import SingleCameraSetup, StereoCameraSetup # Load config cfg = config.read_config() # Create logger logger = logging.getLogger(__name__) logging.basicConfig(format=cfg["log"]["format"], level=cfg["log"]["level"]) def mark_calibration_points(image, image_marker=None, type="checkerboard"): """ Draws the checkerboard corners on the image :param image: the image to process in grayscale :param image_marker: optional. Draws the markers on this image, can be the same as image :return: the corners found """ # Mark calibration points dimension = cfg["calibration"][type]["dimension"] if type == "checkerboard": found, corners = cv.findChessboardCorners(image, dimension, None, flags=cv.CALIB_CB_ADAPTIVE_THRESH) else: found, corners = cv.findCirclesGrid(image, dimension, flags=cv.CALIB_CB_ASYMMETRIC_GRID)
def pipeline(): config_dict = config.read_config() selected_sources = st.sidebar.multiselect('Data Source: ', list(config_dict.keys())) if not selected_sources: st.subheader( "Click the arrow in top left to open the sidebar and select a data source!" ) selected_config = { selected_key: config_dict[selected_key] for selected_key in selected_sources } show_config = st.sidebar.checkbox('Show config', value=False, key='key1') show_load_function_output = st.sidebar.checkbox( 'Show load function output', value=True, key='key2') filtered_region_level = st.sidebar.radio('Filter by region levels', [1, 2, 3, 'All']) for k in selected_config: st.header('Data source: ' + k) params = config_dict[k] data_params = params['data'] data_keys = list(data_params.keys()) filtered_data_types = st.sidebar.multiselect( 'Filter ' + k + ' by data type:', data_keys) if show_config: st.subheader('Config: ') st.write(params) load_func_name = params['load']['function'] if load_func_name == 'None': st.write('No load function') else: load_func = getattr(load_functions, load_func_name) df = load_func(params) if show_load_function_output: st.subheader('Load function output:') st.write(df) st.subheader('Filtered data:') #show_regions = st.checkbox('Filter by region level', value=False, key='key3') #filtered_region_level = st.checkbox('Filter by region level', value=False, key='key3') filtered_df = get_filtered_df(df, data_params, filtered_data_types, filtered_region_level) st.write(filtered_df) plot_data_types = st.sidebar.multiselect( 'Plot ' + k + ' by data type:', data_keys) plot_data_cols = selected_data_types_to_cols(plot_data_types, data_params) unique_regions = df.region_code.unique() selected_regions = st.sidebar.multiselect('Select regions:', unique_regions) for region in unique_regions: st.write(region) region_df = df[df['region_code'] == region] source_df = plot_utils.melt_and_filter_data( plot_data_cols, region_df) st.altair_chart(plot_utils.ruled_altair_chart(source_df), use_container_width=True)
'data/exports/cc_by/LICENSE') EXPORT_PATH_CC_BY_SA_LICENSE = os.path.join(ROOT_DIR, 'data/exports/cc_by_sa/LICENSE') sys.path.append(PIPELINE_DIR) import config cc_by_header = '''The file `aggregated_cc_by.csv` is licensed under Creative Commons Attribution 4.0 International.\n It includes content under the following licenses:\n\n''' cc_by_sa_header = '''The file `aggregated_cc_by_sa.csv` is licensed under Creative Commons Attribution-ShareAlike 4.0 International.\n It includes content under the following licenses:\n\n''' complete_texts = '''Complete license texts for each unique license are available below:\n\n''' config_dict_cc_by = config.read_config(cc_by_sa=False) config_dict_cc_by_sa = config.read_config(cc_by_sa=True) all_license_files_cc_by = ['docs/license_files/cc-by-4.0'] all_license_files_cc_by_sa = ['docs/license_files/cc-by-sa-4.0'] for (key, source_params) in config_dict_cc_by.items(): if 'license' in source_params: license_params = source_params['license'] if 'file' in license_params: path = license_params['file'] if path not in all_license_files_cc_by: all_license_files_cc_by.append(path) print(all_license_files_cc_by) for (key, source_params) in config_dict_cc_by_sa.items():
def main(): config = read_config() c = conn(config, method='post') args = docopt(__doc__, version='%s %s' % (__cli_name__, __version__)) res = False res_get = False if args['deploy']: try: res = deploy_vm(c, args) except Exception as e: print "error deploying instance: %s" % e sys.exit(1) elif args['destroy']: try: id = c.list_virtualmachines(name=args['INSTANCE']) if len(id) == 1: res = destroy_vm(c, id[0].id) else: # Multiple VMs returned if args['--network']: id = c.list_virtualmachines(name=args['INSTANCE'], networkid=get_network( c, args['--network'])[0].id) res = destroy_vm(c, id[0].id) else: print "Multiple instances with name: %s found, please supply a network name" % args[ 'INSTANCE'] except Exception as e: print "Error destroying instance: %s" % e elif args['stop']: try: id = c.list_virtualmachines(name=args['INSTANCE']) if len(id) == 1: res = stop_vm(c, id[0].id) else: # Multiple VMs returned if args['--network']: id = c.list_virtualmachines(name=args['INSTANCE'], networkid=get_network( c, args['--network'])[0].id) res = stop_vm(c, id[0].id) else: print "Multiple instances with name: %s found, please supply a network name" % args[ 'INSTANCE'] except Exception as e: print "Error stopping instance: %s" % e elif args['start']: try: id = c.list_virtualmachines(name=args['INSTANCE']) if len(id) == 1: res = start_vm(c, id[0].id) else: # Multiple VMs returned if args['--network']: id = c.list_virtualmachines(name=args['INSTANCE'], networkid=get_network( c, args['--network'])[0].id) res = start_vm(c, id[0].id) else: print "Multiple instances with name: %s found, please supply a network name" % args[ 'INSTANCE'] except Exception as e: print "Error starting instance: %s" % e elif args['list']: res = list_vms(c, args) elif args['get']: res = c.list_virtualmachines(name=args['INSTANCE']) else: print "Unable to execute command" sys.exit(1) if res: print_tabulate(res, noheader=args['--noheader'], short=args['--short']) else: print "No virtual machines found, deploy new machines using `rbc-instances deploy`" sys.exit(1)
import copy import sys from datetime import datetime from shutil import copyfile from jinja2 import Environment, FileSystemLoader from tridesclous import DataIO, CatalogueConstructor, Peeler import numpy as np from tridesclous import metrics import matplotlib.pyplot as plt import os from config import read_config from spike_sorting.plot import heatmap, annotate_heatmap, plot_clusters_summary cfg = read_config() _cluster_merge_template = """ Cluster {new_cluster_label}, Cell {new_cell_label} Mapped to cluster {old_cluster_label}, Cell {old_cell_label} Similarity : {similarity} """ def run_compare_catalogues(subject, date, similarity_threshold=0.7): bin_min = 0 bin_max = 100 bin_size = 1. bins = np.arange(bin_min, bin_max, bin_size)
import os import glob import threading import train import config if __name__ == '__main__': configurations_dir = "/home/rqa/renato/git/A-Convolutional-Neural-Network-Approach-for-Speech-Quality-Assessment/configurations" if os.path.isdir(configurations_dir): os.chdir(configurations_dir) files = glob.glob("*.json") for filename in files: print('----------------------------------------------') print('Running configuration: ' + filename) print('----------------------------------------------') FLAGS, _unparsed = config.set_flags(config.read_config(filename)) processThread = threading.Thread( target=train.main, args=[[FLAGS, os.path.splitext(filename)[0]]]) processThread.start() processThread.join() print('----------------------------------------------') print('Ending configuration: ' + filename) print('----------------------------------------------') print('')
config.vprint("Upgraded from hamstall file version 4 to 5.") try: file_version = prog_manage.get_file_version('file') except KeyError: file_version = 1 config.write_db() if prog_manage.get_file_version( 'prog' ) == 1: # Online update broke between prog versions 1 and 2 of hamstall print( 'Please manually update hamstall! You can back up your directories in ~/.hamstall !' ) generic.leave() if config.read_config("AutoInstall"): # Auto-update, if enabled prog_manage.update(True) if args.remove_lock: # Argument parsing print("Lock doesn't exist, so not removed!") generic.leave() elif args.install is not None: if not config.exists(args.install): print("File does not exist!") generic.leave() program_internal_name = config.name(args.install) # Get the program name if program_internal_name in config.db["programs"]: # Reinstall check reinstall = generic.get_input( "Application already exists! Would you like to reinstall/overwrite? [r/o/N]", ["r", "o", "n"], "n") # Ask to reinstall
from dataloader import prepare_data, create_dataloader from config import read_config from training import * from sklearn.metrics import accuracy_score, classification_report, matthews_corrcoef parser = argparse.ArgumentParser() parser.add_argument('--config', default="config.ini", help="INI file for model configuration") parser.add_argument( '--model', help="Which model to run", choices=["baseline", "vanilla-bert", "bert-linear", "bert-bilstm"], default="bert-linear") PATHS = read_config(filename="config.ini", section="DATASET") HYPERPARAMS = read_config(filename="config.ini", section="HYPERPARAMS") def predict_labels_baseline(classifier, test_dataloader): model = AutoModel.from_pretrained("allenai/scibert_scivocab_uncased", output_hidden_states=True) model.to(device) test_embeddings = [] test_labels = [] for idx, batch in enumerate(test_dataloader): if idx % 20 == 0: print(f"Done with {idx} of {len(test_dataloader)} batches") b_input_ids = batch[0].to(device)
def main(config_file, num_opt_eval): conf = read_config(config_file) print("config:") pprint(conf) train_df, test_df = get_train_test(conf) feats = [ f for f in train_df.columns if f not in ( ['TARGET', 'SK_ID_CURR', 'SK_ID_BUREAU', 'SK_ID_PREV', 'index']) ] print(f"use {len(feats)} features.") pbar = tqdm(total=num_opt_eval, desc="optimize blend") def objective(params): pbar.update() warnings.simplefilter('ignore') s = sum([v for k, v in params.items() if k in feats]) for p in feats: params[p] = params[p] / s test_pred_proba = pd.Series(np.zeros(train_df.shape[0]), index=train_df.index) for f in feats: test_pred_proba += (train_df[f]**params["power"]) * params[f] score = roc_auc_score(train_df['TARGET'], test_pred_proba) pbar.write(f"power {params['power']}: {score:.6f}") return {'loss': -1.0 * score, 'status': STATUS_OK} parameter_space = {"power": hp.quniform("power", 1, 8, 1)} for c in feats: parameter_space[c] = hp.quniform(c, 0, 1, 0.001) print("====== optimize blending parameters ======") trials = hyperopt.Trials() best = fmin(objective, parameter_space, algo=tpe.suggest, max_evals=num_opt_eval, trials=trials, verbose=1) pbar.close() s = sum([v for k, v in best.items() if k in feats]) for p in feats: best[p] = best[p] / s print("====== best weights to blend ======") pprint(best) # for key, val in best.items(): # print(f" {key}: {val}") print("============= best score =============") best_score = -1.0 * trials.best_trial['result']['loss'] print(best_score) output_directory = os.path.join( conf.dataset.output_directory, f"{datetime.now().strftime('%m%d%H%M')}_{conf.config_file_name}_{best_score:.6f}" ) print(f"write results to {output_directory}") if not os.path.exists(output_directory): os.makedirs(output_directory) with open(os.path.join(output_directory, "result.json"), "w") as fp: results = { "weights": best, "score": best_score, "eval_num": num_opt_eval } print(results) json.dump(results, fp, indent=2) pickle.dump(trials.trials, open(os.path.join(output_directory, f'trials.pkl'), 'wb')) sub_preds = np.zeros(test_df.shape[0]) for f in feats: sub_preds += (test_df[f]**best["power"]) * best[f] test_df['TARGET'] = sub_preds test_df[['SK_ID_CURR', 'TARGET']].to_csv(os.path.join(output_directory, 'submission.csv'), index=False)
import export_utils import doc_utils import config import path_utils args = args_utils.get_parser().parse_args() path_utils.root_dir = args.publish_dir if not args.allowlist: logging.warning( 'RUNNING WITHOUT THE ALLOWLIST! DO NOT MAKE A PULL REQUEST WITH THE OUTPUT!' ) sources_all = config.read_config(cc_by=True, cc_by_sa=True, cc_by_nc=True, google_tos=True, filter_not_approved=args.allowlist) sources_cc_by = config.read_config(cc_by=True, cc_by_sa=False, cc_by_nc=False, google_tos=False, filter_not_approved=args.allowlist) sources_cc_by_sa = config.read_config(cc_by=True, cc_by_sa=True, cc_by_nc=False, google_tos=False, filter_not_approved=args.allowlist) sources_cc_by_nc = config.read_config(cc_by=True, cc_by_sa=False, cc_by_nc=True,
def main(): parser = argparse.ArgumentParser( description='Micro-seismic earthquake detection: "Training" Step') parser.add_argument('--config', required=True, metavar='FILE', dest='config_file', help='configuration file to use') parser.add_argument('--batch-size', type=int, default=32, metavar='N', help='input batch size for training (default: 32)') parser.add_argument('--test-batch-size', type=int, default=100, metavar='N', help='input batch size for testing (default: 100)') parser.add_argument( '--num-workers', type=int, default=1, metavar='W', help='how many subprocesses to use for data loading (default: 1)') parser.add_argument('--epochs', type=int, default=10, metavar='N', help='number of epochs to train (default: 10)') parser.add_argument('--lr', type=float, default=0.0001, metavar='LR', help='learning rate (default: 0.0001)') parser.add_argument('--momentum', type=float, default=0.5, metavar='M', help='SGD momentum (default: 0.5)') parser.add_argument('--no-cuda', dest='use_cuda', action='store_false', default=True, help='disables CUDA training') parser.add_argument( '--train-split-ratio', type=float, default=0.8, metavar='R', help='the portion of the dataset used for training (default: 0.8)') parser.add_argument('--random-split', action='store_true', default=False, help='shuffle the dataset before splitting') parser.add_argument('--seed', type=int, default=None, metavar='S', help='seed Python and Pytorch RNGs (default: None)') parser.add_argument( '--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') parser.add_argument( '--test-model', action='store_true', default=False, help='test final model and save predictions and ground-truth arrays') parser.add_argument('--save-model', action='store_true', default=False, help='save the current (trained) model') parser.add_argument( '--save-costs', action='store_true', default=False, help='write train and test cost values to an ascii column file') parser.add_argument( '--summary', action='store_true', default=False, help='write summarized info on current model to a plain text file') parser.add_argument( '--logfile', action='store_true', default=False, help='write logfile for current run to a plain text file') args = parser.parse_args() # ---------- tic = datetime.now() tic_str = tic.strftime('%Y-%m-%d_%H-%M') # CUDA for PyTorch use_cuda = args.use_cuda and torch.cuda.is_available() device = torch.device("cuda:0" if use_cuda else "cpu") # Set seed for RNGs if args.seed: random.seed(args.seed) torch.manual_seed(args.seed) torch.cuda.manual_seed(args.seed) # for current GPU torch.cuda.manual_seed_all(args.seed) # for all GPUs # Improve performance by enabling benchmarking feature # see: https://pytorch.org/docs/stable/notes/randomness.html cudnn.benchmark = True # ---------- # Read dataset config file config = read_config(args.config_file) images_dirname = pathlib.Path(config.images_dirname) images_paths_pattern = str( images_dirname.joinpath(config.images_paths_pattern)) images_filename_tmpl = str( images_dirname.joinpath(config.images_filename_tmpl)) targets_filename = config.norm_params_filename # ---------- # kwargs for DataLoader dataloader_kwargs = {'num_workers': args.num_workers, 'shuffle': True} if use_cuda: dataloader_kwargs['pin_memory'] = True # Dataset partitioning + prepare targets partition = get_train_test_partition(images_paths_pattern, args.train_split_ratio, args.random_split) train_targets, test_targets = \ get_train_test_targets(targets_filename, partition) # Training-set generator train_loader = DataLoader(dataset=CustomDataset(images_filename_tmpl, partition['train'], train_targets), batch_size=args.batch_size, **dataloader_kwargs) # Test-set generator test_loader = DataLoader(dataset=CustomDataset(images_filename_tmpl, partition['test'], test_targets), batch_size=args.test_batch_size, **dataloader_kwargs) # ---------- # Create the model object model = MultiTaskConvNet().to(device) # Adam optimizer optimizer = optim.Adam(model.parameters(), lr=args.lr) # Loss function class_criterion = nn.BCEWithLogitsLoss() bbox_criterion = nn.MSELoss() train_costs, test_costs = [], [] for epoch in range(1, args.epochs + 1): # Training step train_cost = train(model, optimizer, class_criterion, bbox_criterion, train_loader, device, epoch, args) # Evaluation step test_cost = evaluate(model, class_criterion, bbox_criterion, test_loader, device) print('\nTrain-set epoch cost : {:.9f}'.format(train_cost)) print('Test-set average loss: {:.9f}\n'.format(test_cost)) train_costs.append(train_cost) test_costs.append(test_cost) # ---------- fstem_dated = '{}_{}'.format(g_output_fstem, tic_str) # Save trained model if args.save_model: filename_model = fstem_dated + '_model.pth' torch.save(model.state_dict(), filename_model) # Save cost values if args.save_costs: filename_costs = fstem_dated + '_cost.nc' costs = xr.DataArray(np.vstack((train_costs, test_costs)).T, dims=['epoch_index', 'epoch_cost'], coords={'epoch_cost': ['train', 'test']}) costs.to_netcdf(filename_costs) # Save model predictions if args.test_model: predictions, groundtruth = test(model, test_loader, device, args) predictions.to_netcdf(fstem_dated + '_pred.nc') groundtruth.to_netcdf(fstem_dated + '_true.nc') # ---------- toc = datetime.now() # Write log-file if args.logfile: filename_logfile = fstem_dated + '_log.txt' with open(filename_logfile, 'w') as fid: fid.write('\n'.join(sys.argv[1:])) fid.write('\n') if args.save_model: fid.write('Trained model: {}\n'.format(filename_model)) fid.write('Started at: {}\n'.format(tic)) fid.write('Finished at: {}\n'.format(toc)) # Write model summary if args.summary: dummy_model = MultiTaskConvNet().to(device) filename_summary = fstem_dated + '_summary.txt' original_stdout = sys.stdout with open(filename_summary, 'w') as fid: fid.write(str(dummy_model)) fid.write('\n' * 3) # NOTE: `device` arg should be `str` not `torch.device` sys.stdout = fid images_batch, labels_batch = next(iter(test_loader)) input_shape = images_batch.shape[1:] summary(dummy_model, input_shape, batch_size=args.batch_size, device=device.type) sys.stdout = original_stdout
ch = logging.StreamHandler() ch.setLevel(logging.DEBUG) ## create formatter formatter = logging.Formatter( '%(asctime)s - %(threadName)s - %(name)s - %(levelname)s - %(message)s' ) ## add formatter to ch ch.setFormatter(formatter) ## add ch to logger logger.addHandler(ch) logger.info('Reading configuration.') try: conf = config.read_config() except Exception as e: logger.error('Exception while reading configuration: %s' % e) logger.info('Creating AstroPlant network client.') api_client = astroplant_client.Client(conf["api"]["root"], conf["websockets"]["url"]) logger.info('Authenticating AstroPlant network client.') api_client.authenticate(conf["auth"]["serial"], conf["auth"]["secret"]) logger.info('Initialising kit.') kit = Kit(api_client, conf["debug"]) kit.run()
# # You should have received a copy of the GNU General Public License # along with the sofware; see the file COPYING. If not, write to the # Free Software Foundation, Inc., # 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA. # import __init__ import sys import server, config, config_affairs #-------------------------------------------------------------- # config affairs # look for default config name in lib/config.py conf = config.read_config( config.findConfigFileNameInArgv(sys.argv, __init__.ntlmaps_dir + '/')) conf['GENERAL']['VERSION'] = '0.9.9' #-------------------------------------------------------------- print 'NTLM authorization Proxy Server v%s' % conf['GENERAL']['VERSION'] print 'Copyright (C) 2001-2004 by Dmitry Rozmanov and others.' config = config_affairs.arrange(conf) #-------------------------------------------------------------- # let's run it serv = server.AuthProxyServer(config) serv.run()
#!/usr/bin/python2 import config import sys cfg = dict() if len(sys.argv) > 1: cfg = config.read_config(sys.argv[1]) else: cfg = config.read_config(config.DEFAULT_CONFIG_FILE) scriptfname = "gscript" if len(sys.argv) > 2: scriptfname = sys.argv[2] prefix = cfg['output_prefix'] outdir = cfg['output_dir'] points = open(outdir + "/points_" + prefix + "_all") size = len(points.readlines()) points.close() script = "set terminal png size 2048,2048\n" bigscript = "plot \"" + outdir + "/lines_" + prefix + "_all\" with lines " for i in range(0,size): script = script + "set output \"" + outdir + "/chunks_" + prefix + "_" + str(i) + ".png\"\n" script = script + "plot \"" + outdir + "/points_" + prefix + "_" + str(i) + "\" with points, \"" + outdir + "/chunks_" + prefix + "_" + str(i) + "\" with lines\n" script = script + "set output \"" + outdir + "/chunks_raw_" + prefix + "_" + str(i) + ".png\"\n" script = script + "plot \"" + outdir + "/points_" + prefix + "_" + str(i) + "\" with points" script = script + ",\"" + outdir + "/chunks_raw_" + prefix + "_" + str(i) + "\" with lines\n" bigscript = bigscript + ",\"" + outdir + "/points_" + prefix + "_" + str(i) + "\" with points, \"" + outdir + "/chunks_" + prefix + "_" + str(i) + "\" with lines"