示例#1
0
  def test_ignore_long_suspensions(self):
    # Ensure that we have a known value for when we start ignoring plans.
    conf = Config()
    conf.PLAN_USER_IGNORE_THRESHOLD = 30

    self.plan1.member_limit = 1

    user = Membership(first_name="Testy", last_name="Testerson",
                      email="*****@*****.**", plan="plan1",
                      status="active")
    user.put()

    # Initially, the plan should be full, for every status.
    self.assertTrue(self.plan1.is_full())
    user.status = "suspended"
    user.put()
    self.assertTrue(self.plan1.is_full())
    user.status = None
    user.put()
    self.assertTrue(self.plan1.is_full())

    # If we mess with the updated time, it should be ignored when the plan is
    # not active.
    user.updated = datetime.datetime.now() - datetime.timedelta(days=31)

    user.status = "active"
    user.put(skip_time_update=True)
    self.assertTrue(self.plan1.is_full())
    user.status = "suspended"
    user.put(skip_time_update=True)
    self.assertFalse(self.plan1.is_full())
    user.status = None
    user.put(skip_time_update=True)
    self.assertFalse(self.plan1.is_full())
示例#2
0
    def __init__(self):
        QtGui.QMainWindow.__init__(self)
        self.main =  Ui_MainWindow()
        self.flowconfig = flowconfig.FlowConfig(self)        
        self.app = None
        self.streammod = StreamTable(self)

        # Dialogs
        self.aboutdialog = AboutDialog.AboutDialog(self) 
                
        # Initialized after setupUi runs
        self.interfacesgui = None
        self.protocolsgui = None
        self.rulegui = None
        self.dbgui = None
        
        debugger_uri = "PYROLOC://127.0.0.1:7766/debugger"
        self.remote_debugger = Pyro.core.getProxyForURI(debugger_uri)
        
        #self.proxy = xmlrpclib.ServerProxy("http://localhost:20757")
        #self.objectproxy = xmlrpclib.ServerProxy("http://localhost:20758")
        self.curdebugevent = ""

        self.log = logging.getLogger("mallorygui")
        config = Config()
        config.logsetup(self.log)
示例#3
0
文件: tests.py 项目: caryt/utensils
class TestConfig(TestCase):
    """Test :class:`.Config`."""

    def setUp(self):
        environ["CONFIG_OPTION"] = "override"
        self.config = Config()

    def test_defaults(self):
        """Test default configuration loaded from defaults.py."""
        self.assertEqual(self.config.CONFIG_NAME, 'value')

    def test_overrides(self):
        """Test configuration loaded from environment variables."""
        self.assertEqual(self.config.CONFIG_OPTION, 'override')

    def test_iter(self):
        """Test iterating over configuration."""
        self.assertEqual([config for config in self.config],
            [('CONFIG_NAME', 'value'), ('CONFIG_OPTION', 'override')]
        )

    def test_json(self):
        """Test JSON configuration."""
        self.assertEqual(self.config.json(),
            ['CONFIG_NAME = value', 'CONFIG_OPTION = override']
        )

    def test_html(self):
        """Test HTML configuration."""
        self.assertEqual(self.config.html(),
            '<ul><li>CONFIG_NAME = value<li>CONFIG_OPTION = override</ul>'
        )
示例#4
0
文件: cli.py 项目: eirikba/ebakup
def make_tasks_from_args(args):
    localtree = args.services['filesystem']('local')
    config = Config(args.services)
    if args.config:
        config.read_file(localtree, args.config)
    else:
        confpaths = localtree.get_config_paths_for('ebakup')
        for path in confpaths:
            config.read_file(localtree, path + ('config',))
    tasks = []
    tasks.append(WebUITask(config, args))
    if args.command == 'backup':
        task = BackupTask(config, args)
        tasks.append(task)
    elif args.command == 'info':
        task = InfoTask(config, args)
        tasks.append(task)
    elif args.command == 'shadowcopy':
        task = MakeShadowTreeTask(config, args)
        tasks.append(task)
    elif args.command == 'webui':
        pass
    elif args.command == 'sync':
        task = SyncTask(config, args)
        tasks.append(task)
    elif args.command == 'verify':
        task = _make_verify_task(config, args)
        tasks.append(task)
    else:
        raise UnknownCommandError('Unknown command: ' + args.command)
    return tasks
示例#5
0
 def settype(self, name, type):
     """ set the type of a bot. """
     cfg = Config('fleet' + os.sep + stripname(name) + os.sep + 'config')
     cfg['name'] = name
     logging.debug("%s - setting type to %s" % (self.cfile, type))
     cfg.type = type
     cfg.save()
示例#6
0
def main():
    settings = {
        'cookie_secret': '66oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=',
        'login_url':  r'/login',
        'debug': True,
    }

    #load config xml and init logger
    conf = Config()
    conf.load('./wm.xml')
    LOG().setlogger('./wm.log')
    print conf.to_string()

    html_mgn().start(conf.bnum)
    wx_mongo.WxMongo().start()

    omgn = order_mgn.OrderMgn()
    omgn.init()
    onotify = order_notify.OrderNotify()
    onotify.start(wx_mongo.WxMongo(), conf)
    discount_mgn.DiscountMgn().start('./dc.xml')
    print 'server ip[%s],port[%d]' % (conf.ip, conf.port)
    # Start server
    app = tornado.web.Application([
        (r'/wm', WMHandler),
        (r'/openid', OrderHandler),
        (r'/shopping', ShoppingHandler),
        (r'/test', TestHandler),
        ], **settings)

    app.listen(conf.port)
    tornado.ioloop.IOLoop.instance().start()
示例#7
0
	def getTodos(self):
		cfg = Config()
		self.aleph.login(*[str(cfg.value('aleph_loaned/' + key).toString()) for key in CFGS])
		try:
			return map(self.loan2entry, self.aleph.get_loaned())
		finally:
			self.aleph.logout()
示例#8
0
文件: sncli.py 项目: Qu4tro/sncli
class sncli:

    def __init__(self, do_server_sync, verbose=False):
        self.config         = Config()
        self.do_server_sync = do_server_sync
        self.verbose        = verbose
        self.do_gui         = False

        if not os.path.exists(self.config.get_config('db_path')):
            os.mkdir(self.config.get_config('db_path'))

        # configure the logging module
        self.logfile = os.path.join(self.config.get_config('db_path'), 'sncli.log')
        self.loghandler = RotatingFileHandler(self.logfile, maxBytes=100000, backupCount=1)
        self.loghandler.setLevel(logging.DEBUG)
        self.loghandler.setFormatter(logging.Formatter(fmt='%(asctime)s [%(levelname)s] %(message)s'))
        self.logger = logging.getLogger()
        self.logger.setLevel(logging.DEBUG)
        self.logger.addHandler(self.loghandler)
        self.config.logfile = self.logfile

        logging.debug('sncli logging initialized')

        self.logs = []

        try:
            self.ndb = NotesDB(self.config, self.log, self.gui_update_view)
        except Exception, e:
            self.log(str(e))
            sys.exit(1)
示例#9
0
class Circulation(object):
    '''
    Makes a connection with the WMS Circulation API using the classes Connect (originally from OCLC) and Config
    '''
    def __init__(self, service=None, useTestData = True):
        service = service # 'circulation' # WMS Circulation API
        self.config = Config(service = service, useTestData = useTestData)
        self.institution = self.config.getInstitutionData()
        self.serviceAPI = self.config.getServiceAPIData()
        self.institutionList = self.institution.get('institution')
        self.branchList = self.institution.get('branch')

    def get_data_response(self, institution_id = None, request_url = ''):
        connect = Connect(institution_id = institution_id, request_url = request_url, config = self.config)
        opener = urllib2.build_opener(MyHTTPSHandler)
        opener.addheaders = [('accept', 'application/json'),('Authorization', connect.getAuthorization_header())]
        response = opener.open(request_url).read()
        return response

    def get_JSON(self, data):
        _data = data
        try:
            return json.loads(_data)
        except Exception, inst:
            print inst
示例#10
0
 def get_feed_urls(self):
     ''' Open urls file in .config, make list of feeds '''
     feeds = []
     try:
         feedfile = open(CONFIGPATH + 'urls')
     except Exception as e:
         feedfile = Config().create_urls()
     if feedfile is True:
         print(f"Add urls to url file at: {CONFIGPATH + 'urls'}")
         sys.exit
     else:
         for line in feedfile:
             line = line.strip()
             if not line.startswith("#"):
                 line = line.replace('\n', '').strip()
                 line = line.split('|')
                 try:
                     genre = re.sub(r'[-\s]+', '-', (re.sub(r'[^\w\s-]', '', line[1]).strip().lower()))
                 except Exception as e:
                     genre = 'uncategorized'
                 if line[0]:
                     feed = line[0].strip()
                     db.add_url(feed)
                     feeds += [[feed, genre]]
                     self.get_media_links(feed, genre)
                     now = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
                     db.update_time(feed, now)
         feedfile.close()
         return True
示例#11
0
    def makebot(self, name, cfg=None):

        """ create a bot with name .. use configuration is provided. """

        bot = None

        # if not config create a default bot
        if not cfg:
            cfg = Config(self.datadir + os.sep + name, 'config', inittxt=fleetbotconfigtxt)
            cfg.save()

        # create bot based on type 
        if cfg['type'] == 'irc':
            from gozerbot.irc.bot import Bot
            bot = Bot(cfg)
        elif cfg['type'] == 'jabber':
            from gozerbot.jabber.jabberbot import Jabberbot
            bot = Jabberbot(cfg)
        else:
            logging.debug('unproper type: %s' % cfg['type'])

        # set bot name and initialize bot
        if bot:
            cfg['name'] = bot.name = name
            self.initbot(bot)
            return bot

        # failed to created the bot
        raise Exception("can't make %s bot" % name)
示例#12
0
文件: main.py 项目: bjpop/lynch_gatk
def main():
    '''Initialise the pipeline, then run it'''
    # Parse command line arguments
    options = parse_command_line()
    # Initialise the logger
    logger = Logger(__name__, options.log_file, options.verbose)
    # Log the command line used to run the pipeline
    logger.info(' '.join(sys.argv))
    drmaa_session = None
    try:
        # Set up the DRMAA session for running cluster jobs
        import drmaa
        drmaa_session = drmaa.Session()
        drmaa_session.initialize()
    except Exception as e:
        print("{progname} error using DRMAA library".format(progname=program_name), file=sys.stdout)
        print("Error message: {msg}".format(msg=e.message, file=sys.stdout))
        exit(error_codes.DRMAA_ERROR)
    # Parse the configuration file, and initialise global state
    config = Config(options.config)
    config.validate()
    state = State(options=options, config=config, logger=logger,
                  drmaa_session=drmaa_session)
    # Build the pipeline workflow
    pipeline = make_pipeline(state)
    # Run (or print) the pipeline
    cmdline.run(options)
    if drmaa_session is not None:
        # Shut down the DRMAA session
        drmaa_session.exit()
示例#13
0
def run(args):
    """
    run the corresponding action
    """
    my_config = Config()
    if args['action'] == 'config':
        args['list_repositories'] and my_config.print_repositories()
    else:
        module_obj = Module(
            args['module_name'], args['module_developers'],
            args['module_planners'],
            args['module_auditors'], folder=args['destination_folder'],
            init_data=args['add_init_data'], company_name=args['company_name'])
        if args['action'] == 'branch':
            branch_obj = Branch(
                module_obj, args['branch_suffix'], args['parent_repo'],
                args['oerp_version'], args['destination_folder'])
            branch_obj.create_branch()
            module_obj.create(branch_obj)
        elif args['action'] == 'create':
            module_obj.create()
        elif args['action'] == 'append':
            module_obj.append(args['append_file'], args['file_name'])

        #~ module_obj.branch_changes_apply()
    return True
示例#14
0
 def createConfig(self):
     config = Config()
     config.executionTime=1
     config.programSpeed=2
     config.maxTravel=3
     config.formulae=["4", "5"]
     return config
示例#15
0
def load_config(default_config='train_sda.cfg', reset_logging=True):
    parser = argparse.ArgumentParser(description='Train a Stacked Denoising Autoencoder');
    parser.add_argument('-c', '--config', help='specify a configuration file');
    parser.add_argument('-v', '--verbose', help='increase output verbosity", action="store_true');    
    args = parser.parse_args();
    
    if args.config == None:
        configfile = default_config; # load default config
    else:
        configfile = args.config;
    config = Config(file(configfile));  
    
    if args.verbose or config.logger.level=='debug':
        loglevel = logging.DEBUG;
    else:
        loglevel = logging.INFO;
    
    logging.basicConfig(format=config.logger.pattern, level=loglevel);    

    if reset_logging or config.get('reset_logging', True):
        reset_pylearn2_logging();
        
    logging.info('using config {0}'.format(configfile));
    
    # disable annoying deprecation warnings
    warnings.simplefilter('once', UserWarning)
    warnings.simplefilter('default')
    
    return config;
示例#16
0
    def __init__(self, id=None, **kwargs):
        Config.__init__(self)
        self._cur_event = None
        self._last_event = 0
        self._rebuild_dh_params = False
        self._reset_ip_pool = False

        if id is None:
            self.id = uuid.uuid4().hex
            for name, value in kwargs.iteritems():
                setattr(self, name, value)
        else:
            self.id = id

        self.path = os.path.join(app_server.data_path, SERVERS_DIR, self.id)
        self.ovpn_conf_path = os.path.join(self.path, TEMP_DIR, OVPN_CONF_NAME)
        self.dh_param_path = os.path.join(self.path, DH_PARAM_NAME)
        self.ip_pool_path = os.path.join(self.path, IP_POOL_NAME)
        self.ca_cert_path = os.path.join(self.path, TEMP_DIR, OVPN_CA_NAME)
        self.tls_verify_path = os.path.join(self.path, TEMP_DIR,
            TLS_VERIFY_NAME)
        self.user_pass_verify_path = os.path.join(self.path, TEMP_DIR,
            USER_PASS_VERIFY_NAME)
        self.client_connect_path = os.path.join(self.path, TEMP_DIR,
            CLIENT_CONNECT_NAME)
        self.client_disconnect_path = os.path.join(self.path, TEMP_DIR,
            CLIENT_DISCONNECT_NAME)
        self.ovpn_status_path = os.path.join(self.path, TEMP_DIR,
            OVPN_STATUS_NAME)
        self.auth_log_path = os.path.join(app_server.data_path, AUTH_LOG_NAME)
        self.set_path(os.path.join(self.path, SERVER_CONF_NAME))

        if id is None:
            self._initialize()
示例#17
0
 def init_skin(self, skin_name, system_skin_dir, user_skin_dir, skin_config_file,
               app_given_id, app_given_version):
     '''Init skin.'''
     self.skin_config_file = skin_config_file
     if os.path.exists(skin_config_file):
         # Read skin name from config file.
         skin_config = Config(skin_config_file)
         skin_config.load()
         
         # Load skin.
         init_skin_name = skin_config.get("skin", "skin_name")
     else:
         # Create skin config if it not exists.
         touch_file(self.skin_config_file)
         
         init_skin_name = skin_name
         
     if self.is_skin_exist(init_skin_name, system_skin_dir, user_skin_dir):
         self.load_skin(init_skin_name, system_skin_dir, user_skin_dir)
     else:
         # Try load default skin if user's select skin not exists.
         default_skin_name = self.get_default_skin(system_skin_dir, user_skin_dir)
         assert(default_skin_name != None)
         self.load_skin(default_skin_name, system_skin_dir, user_skin_dir)
         
     self.app_given_id = app_given_id
     self.app_given_version = app_given_version
示例#18
0
def create_protos():
    """Create a Protocol instance with some fields."""
    dissector.Protocol.protocols = {}
    conf = Config('tester')
    conf.id = [25,]
    conf.description = 'This is a test'

    rules = [Trailer(conf, {'name': 'missing', 'member': 'missing', 'size': 0}),
             Trailer(conf, {'name': 'simple', 'count': 1, 'size': 4}),
             Trailer(conf, {'name': 'bur', 'count': 3, 'size': 8}),
             Trailer(conf, {'name': 'ber', 'member': 'count'})]

    proto, diss = dissector.Protocol.create_dissector('tester', None, conf)
    diss.add_field(Field('one', 'float', 4, 0, Platform.big))
    diss.add_field(Field('range', 'float', 4, 0, Platform.big))
    diss.children[-1].set_range_validation(0, 10)
    field = Field('array', 'float', 4, 0, Platform.big)
    diss.add_field(ArrayField.create([1, 2, 3], field))
    field = Field('str', 'string', 30, 0, Platform.big)
    diss.add_field(ArrayField.create([2], field))
    diss.add_field(Field('count', 'int32', 4, 0, Platform.big))
    diss.push_modifiers()
    yield proto
    dissector.Protocol.protocols = {}
    del proto, diss
示例#19
0
    def __init__(self, config=None, envFiles=None):
        self.envFilePaths = envFiles
        self.env = dict()
        self.msgSender = "EnvironmentLayout"

        if self.envFilePaths == None:
            for key, value in os.environ.iteritems():
                self.env[key] = value
        else:
            assert config != None
            if type(self.envFilePaths) != list:
                self.envFilePaths = [self.envFilePaths]

            # sysEnvPaths = config.getEnvPaths()
            sysEnvPaths = list()
            sysEnvPaths.extend(config.getEnvPaths())

            self.__resolveENVFileLocations(sysEnvPaths)

            for path in self.envFilePaths:
                cfg = Config(path)
                if not cfg.has_section("env"):
                    userErrorExit("Missing mandatory section '%s'" % ("env"))

                envDict = cfg.get_section("env")
                self.__mergeEnv(envDict)
示例#20
0
def main():
    """
    Initiates all components that are needed after arduino-yun's linuino
    has started.
    """
    threads = []

    # initiate all needed threads
    data_thread = Data()
    socket_thread = Websocket(
        Config.get_socket_host(),
        Config.get_socket_port(),
        data_thread
    )
    threads.append(data_thread)
    threads.append(socket_thread)

    # start all needed threads
    for thread in threads:
        thread.daemon = True
        thread.start()

    # add observers to data-object
    logger = Logger("default")
    data_thread.add_observer(logger)
    data_thread.add_observer(socket_thread)

    # keep main() alive to prevent an exit of our
    # deamonized threads
    while True:
        time.sleep(1)
示例#21
0
文件: btmanager.py 项目: onze/Weld
    def ifile2dict(self, ifile_fullpath, parent_dirname='unknown'):
        """
        Returns a dict of the attributes contained in the info_file pointed
        by the given fullpath.
        """
        print 'ifile2dict on', ifile_fullpath

        # print 'ifile2xml() on ', ifile_fullpath
        ifile_path, ifile_name = os.path.split(ifile_fullpath)
        
        # once again, make sure the requested class exists
        if not ifile_name in (Config.instance().BT_nodes_classes_names \
                              #+ Config.instance().BT_leaves_classes_names\
                              ):
            s = 'in BTManager.ifile2dict(): unrecognized class \'%s\''
            raise Exception(s % ifile_name)

        with open(ifile_fullpath) as file:
            lines = map(str.strip, file.readlines())
        # skip empty lines
        lines = [line for line in lines
            if line]

        # skip comments
        lines = [line for line in lines
            if not (line.startswith('#') or line.startswith('//'))]

        content = {
            'class':ifile_name,
        }
        
        if lines:
            # split attribute names and values
            splits = [([line, 'True'], line.split('='))[int('=' in line)]
                for line in lines]
            attributes, values = zip(*[(split[0], '='.join(split[1:]))
                                     for split in splits])
            
            # make sure no attr starts with '__'
            for attr in attributes:
                if attr.startswith('__'):
                    s = 'WARNING: in BTManager.ifile2dict(ifile_fullpath=%(ifile_fullpath)s, '\
                        'parent_dirname=%(parent_dirname)s):\nattributes\' names '\
                        'starting with \'__\' are not recommended.'
                    print >> sys.__stderr__, s

            # interpret values as python values if possible
            def cast_or_str(v):
                try:return ast.literal_eval(v)
                except:return v
            values = map(cast_or_str, values)

            content.update(dict(zip(attributes, values)))

        if Config.instance().BT_serialization_add_debug_info:
            content['__parent_dir_name'] = parent_dirname
            content['__ifile_path'] = ifile_path
            content['__ifile_name'] = ifile_name

        return content
示例#22
0
  def __init__(self, settings_path, dependencies_path):
    Config.__init__(self, settings_path, dependencies_path)

    self.put('data_dir', '%s/data/%s' % (self.smt_semparse, self.corpus))

    if self.np:
      self.train_name = 'train.np'
    else:
      self.train_name = 'train'

    self.put('srilm_ngram_count', '%s/bin/%s/ngram-count' % \
                                 (self.srilm, self.srilm_arch))

    self.put('moses_train', '%s/scripts/training/train-model.perl' % self.moses)
    self.put('moses_tune', '%s/scripts/training/mert-moses.pl' % self.moses)
    self.put('moses_decode_phrase', '%s/bin/moses' % self.moses)
    self.put('moses_decode_hier', '%s/bin/moses_chart' % self.moses)
    self.put('bleu_eval', '%s/scripts/generic/multi-bleu.perl' % self.moses)

    self.put('wasp_eval', '%s/data/geo-funql/eval/eval.pl' % self.wasp)

    if self.nlg:
      self.put('src', 'mrl')
      self.put('tgt', 'nl')
    else:
      self.put('src', 'nl')
      self.put('tgt', 'mrl')
示例#23
0
def iter_list():

    cfg = Config()

    methods = cfg.get_methods()
    features = cfg.get_features()
    c_iteration = cfg.get_c_iteration()
    def_c_value = cfg.get_def_c_value()

    i_list = []

    for i in methods:
        if c_iteration is True:
            for j in range(-4,5):
                for k in features:
                    i_list.append({
                            'method': i,
                            'c_value': pow(10, j),
                            'feature': k
                        })
        else:
            for j in features:
                i_list.append({
                        'method': i,
                        'c_value': def_c_value,
                        'feature': j
                    })
    return i_list
示例#24
0
 def clear_cache(self):
     cache_db.set_remove('servers', '%s_%s' % (self.id, self.type))
     cache_db.list_remove('servers_sorted', '%s_%s' % (self.id, self.type))
     cache_db.remove(self.get_cache_key('clients'))
     for period in ('1m', '5m', '30m', '2h', '1d'):
         persist_db.remove(self.get_cache_key('bandwidth-%s' % period))
     Config.clear_cache(self)
示例#25
0
def load_config():
    """
    Load configuration
    """

    parser = argparse.ArgumentParser(description='Configure the change tracking app.')

    parser.add_argument('--sync_cmd', type=str, action='store', nargs='?',
                        help='The command line to sync once there is change', default=Config.DEFAULT_SYNC_CMD)

    parser.add_argument('--delay', type=float, action='store', nargs='?',
                        help='Number of seconds to batch for sync', default=Config.DEFAULT_DELAY)

    parser.add_argument('--filter_regexp', type=str, action='store', nargs='?',
                        help='Regexp for filtering files from sync', default=Config.DEFAULT_FILTER_REGEXP)

    parser.add_argument('--watch_dir', type=str, action='store', nargs='?',
                        help='Path of the dir to watch', default=os.getcwd())

    args = parser.parse_args()

    # New config
    conf = Config()

    # update conf
    conf.delay = args.delay
    conf.sync_cmd = args.sync_cmd
    conf.filter_regexp = args.filter_regexp
    conf.directory = args.watch_dir

    # return
    return conf
示例#26
0
def validate_conf():
    config = Config()
    config.validate_config()
    if not config:
        return jsonify({'error': "Invalid config file"})
    else:
        return jsonify({'result': 'Configuration Loaded'})
示例#27
0
    def __init__(self, id=None, name=None):
        Config.__init__(self)

        if id is None:
            self._initialized = False
            self.id = uuid.uuid4().hex
        else:
            self._initialized = True
            self.id = id
        data_path = app_server.data_path or DEFAULT_DATA_PATH
        self.path = os.path.join(data_path, ORGS_DIR, self.id)

        self.index_path = os.path.join(self.path, INDEX_NAME)
        self.index_attr_path = os.path.join(self.path, INDEX_NAME + '.attr')
        self.serial_path = os.path.join(self.path, SERIAL_NAME)
        self.crl_path = os.path.join(self.path, CRL_NAME)
        self.set_path(os.path.join(self.path, 'ca.conf'))

        if name is not None:
            self.name = name

        if not self._initialized:
            self._initialize()

        self.ca_cert = User(self, id=CA_CERT_ID)
示例#28
0
def main():
    bus = can.interface.Bus(channel=Config.getChannel(), bustype=Config.getBusType())
    print('start_transmit()')
    sendMotorStatusMessage()
    sendEnvironmentStatusMessage()
    while True:
        time.sleep(20)
示例#29
0
    def retry_clone(git_repo):
        """Retry 'git clone' operation for defined number of attempts with defined intervals
        """
        if os.path.isdir(git_repo.local_repo_path) and os.listdir(git_repo.local_repo_path) != []:
            # delete and recreate local repo path if not empty dir
            AgentGitHandler.log.debug("Local repository path not empty. Cleaning.")
            GitUtils.delete_folder_tree(git_repo.local_repo_path)
            GitUtils.create_dir(git_repo.local_repo_path)

        git_clone_successful = False
        # Read properties from agent.conf
        max_retry_attempts = int(Config.read_property(constants.ARTIFACT_CLONE_RETRIES, 5))
        retry_interval = int(Config.read_property(constants.ARTIFACT_CLONE_INTERVAL, 10))
        retry_attempts = 0

        # Iterate until git clone is successful or reaches max retry attempts
        while git_clone_successful is False and retry_attempts < max_retry_attempts:
            try:
                retry_attempts += 1
                Repo.clone_from(git_repo.repo_url, git_repo.local_repo_path)
                AgentGitHandler.add_repo(git_repo)
                AgentGitHandler.log.info(
                    "Retrying attempt to git clone operation for tenant %s successful" % git_repo.tenant_id
                )
                git_clone_successful = True

            except GitCommandError as e:
                AgentGitHandler.log.warn("Retrying git clone attempt %s failed" % retry_attempts)
                if retry_attempts < max_retry_attempts:
                    time.sleep(retry_interval)
                    pass
                else:
                    raise GitRepositorySynchronizationException("Error while retrying git clone: %s" % e)
示例#30
0
    def test_GetRomCollectionNames(self):
        config_xml_file = os.path.join(os.path.dirname(__file__), 'testdata', 'config', 'romcollections_two_valid.xml')
        conf = Config(config_xml_file)
        conf.readXml()

        list = conf.getRomCollectionNames()
        self.assertEqual(list, ['Atari 2600', 'NES'])
def run():
    category_frames = read_native(
        Config.get("feature_text_names.extraction.output.path"), 2000)
    category_frames = filter_frames(
        category_frames,
        get_labeled_data_filter("../../mturk/our_labels/labels.csv"))

    train_frame, test_frame = split_train_test(category_frames, test_size=0.3)

    mask = np.asarray(np.ones((1, train_frame.shape[1]), dtype=bool))[0]
    mask[0] = False

    train_matrix, test_matrix = dataframe_to_numpy_matrix(
        train_frame, test_frame, mask)

    train_x, train_y = split_target_from_data(train_matrix)
    test_x, test_y = split_target_from_data(test_matrix)

    my_stop_words = read_stop_words()
    print my_stop_words

    stop_words = text.ENGLISH_STOP_WORDS.union(my_stop_words)

    pipeline = Pipeline([
        ('vect', CountVectorizer(stop_words=stop_words)),
        ('tfidf', TfidfTransformer()),
        #('clf', SGDClassifier(loss='log')),
        ('clf',
         OneVsRestClassifier(
             SVC(kernel="linear", class_weight='balanced', probability=True))),
    ])
    # uncommenting more parameters will give better exploring power but will
    # increase processing time in a combinatorial way
    parameters = {
        'vect__max_df': (0.5, 0.75, 1.0),
        # 'vect__max_features': (None, 5000, 10000, 50000),
        'vect__ngram_range': ((1, 1), (1, 2)),  # unigrams or bigrams
        # 'tfidf__use_idf': (True, False),
        # 'tfidf__norm': ('l1', 'l2'),
        'clf__alpha': (0.00001, 0.000001, 0.0000001),
        'clf__penalty': ('l2', 'elasticnet')
        # 'clf__n_iter': (10, 50, 80),
    }
    # find the best parameters for both the feature extraction and the
    # classifier
    grid_search = GridSearchCV(pipeline, parameters, n_jobs=4, verbose=10)
    """
    cv = StratifiedShuffleSplit(n_splits=5, test_size=0.2)
    grid_search = GridSearchCV(pipeline, param_grid=parameters, cv=cv, scoring='f1_weighted', verbose=10, n_jobs=4)



    print("Performing grid search...")
    print("pipeline:", [name for name, _ in pipeline.steps])
    print("parameters:")
    pprint(parameters)
    t0 = time()
    grid_search.fit(train_x.A.ravel(), np.array(train_y).tolist())
    print("done in %0.3fs" % (time() - t0))
    print()

    print("Best score: %0.3f" % grid_search.best_score_)
    print("Best parameters set:")
    best_parameters = grid_search.best_estimator_.get_params(deep=True)
    for param_name in sorted(parameters.keys()):
        print("\t%s: %r" % (param_name, best_parameters[param_name]))
    """
    '''
     best_parameters = {
        'vect__max_df': (0.5),
        # 'vect__max_features': (None, 5000, 10000, 50000),
        'vect__ngram_range': ((1, 2)),  # unigrams or bigrams
        # 'tfidf__use_idf': (True, False),
        # 'tfidf__norm': ('l1', 'l2'),
        'clf__alpha': (0.00001),
        'clf__penalty': ('elasticnet')
        # 'clf__n_iter': (10, 50, 80),
    }

    best_parameters = {
        'vect__max_df': (0.5),
        # 'vect__max_features': (None, 5000, 10000, 50000),
        'vect__ngram_range': ((1, 1)),  # unigrams or bigrams
        # 'tfidf__use_idf': (True, False),
        # 'tfidf__norm': ('l1', 'l2'),
        'clf__alpha': (0.0001),
        'clf__penalty': ('elasticnet')
        # 'clf__n_iter': (10, 50, 80),
    }
    '''

    # pipeline.set_params(**best_parameters)

    X_numpy = train_x
    y_numpy = train_y

    X = X_numpy.A.ravel()
    y = np.array(y_numpy).tolist()

    parameters = {'clf__estimator__C': np.logspace(-4, 4, 9)}
    clf = fit_cv(pipeline, X, y, parameters)
    # clf = pipeline.fit(X, y)

    # feature visualization
    vec = clf.named_steps['vect']
    vocabulary = vec.vocabulary_

    classifier = clf.named_steps['clf']
    weights = classifier.coef_.toarray()

    category_list = {
        0: "DATA",
        1: "EDU",
        2: "HW",
        3: "DOCS",
        4: "DEV",
        5: "WEB"
    }

    k = 20
    for id in range(0, len(category_list)):
        max_ids = np.argsort(-weights[id])
        values = np.sort(-weights[id])

        feature_names = [""] * k
        feature_weights = [0] * k
        for key_d, item in vocabulary.iteritems():
            for i in range(0, k):
                if max_ids[i] == item:
                    feature_names[i] = key_d
                    feature_weights[i] = -values[i]

        print feature_names
        array_to_itemlist(feature_names)
        print feature_weights

        ind = np.arange(k)
        plt.bar(ind, feature_weights)
        plt.title('top ' + str(k) + ' features for ' + category_list[id])
        plt.xticks(np.arange(k) + 0.5, feature_names, rotation='vertical')
        plt.xlabel('features')
        plt.ylabel('weight')
        plt.tight_layout()
        # plt.show()

    # y_pred = clf.predict(test_x.A.ravel())

    # y_pred_proba = clf.predict_proba(test_x.A.ravel())
    # print y_pred_proba
    # print "Test Set"
    # visualize(test_y, y_pred)
    test(clf, test_x.A.ravel(), np.array(test_y, dtype=np.float))

    print "Atachment A"
    # attachment A
    attachment_a_frames = read_native(
        Config.get("attachmentA.feature_text_names.extraction.output.path"),
        150)
    attachment_a_frame = concat(attachment_a_frames)
    attachment_a_matrix = dataframe_to_numpy_matrix_single(
        attachment_a_frame, mask)
    attachment_a_x, attachment_a_y = split_target_from_data(
        attachment_a_matrix)
    attachment_a_y_pred = clf.predict_proba(attachment_a_x.A.ravel())
    # print attachment_a_y_pred
    # attachment_a_ypred_test = np.argmax(attachment_a_y_pred, axis=1)  # choose class with highest probability per sample
    # print attachment_a_ypred_test
    # visualize(attachment_a_y, attachment_a_ypred_test)
    test(clf, attachment_a_x.A.ravel(), np.array(attachment_a_y,
                                                 dtype=np.float))
示例#32
0
def genJob(jobPath, testPath, logPath, pathConf):
    f = file(pathConf)
    cfg = Config(f)

    pathToJob = jobPath + "/launchOutStats.pbs"
    if os.path.exists(pathToJob):
        os.system("rm " + pathToJob)

    AllTile = cfg.chain.listTile
    nbTile = len(AllTile.split(" "))

    if nbTile > 1:
        jobFile = open(pathToJob, "w")
        jobFile.write('#!/bin/bash\n\
#PBS -N outStats\n\
#PBS -J 0-%d:1\n\
#PBS -l select=1:ncpus=1:mem=10000mb\n\
#PBS -l walltime=02:00:00\n\
#PBS -o %s/outStats_out.log\n\
#PBS -e %s/outStats_err.log\n\
\n\
\n\
module load python/2.7.5\n\
module remove xerces/2.7\n\
module load xerces/2.8\n\
module load gdal/1.11.0-py2.7\n\
\n\
FileConfig=%s\n\
export ITK_AUTOLOAD_PATH=""\n\
export OTB_HOME=$(grep --only-matching --perl-regex "^((?!#).)*(?<=OTB_HOME\:).*" $FileConfig | cut -d "\'" -f 2)\n\
. $OTB_HOME/config_otb.sh\n\
export ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS=1\n\
\n\
PYPATH=$(grep --only-matching --perl-regex "^((?!#).)*(?<=pyAppPath\:).*" $FileConfig | cut -d "\'" -f 2)\n\
Nsample=$(grep --only-matching --perl-regex "^((?!#).)*(?<=runs\:).*" $FileConfig | cut -d "\'" -f 2)\n\
cd $PYPATH\n\
\n\
ListeTuile=($(grep --only-matching --perl-regex "^((?!#).)*(?<=listTile\:).*" $FileConfig | cut -d "\'" -f 2))\n\
\n\
python outStats.py -tile ${ListeTuile[${PBS_ARRAY_INDEX}]} -conf $FileConfig --sample $Nsample --wd $TMPDIR'
                      % (nbTile - 1, logPath, logPath, pathConf))
        jobFile.close()
    else:
        jobFile = open(pathToJob, "w")
        jobFile.write('#!/bin/bash\n\
#PBS -N outStats\n\
#PBS -l select=1:ncpus=1:mem=4000mb\n\
#PBS -l walltime=02:00:00\n\
#PBS -o %s/outStats_out.log\n\
#PBS -e %s/outStats_err.log\n\
\n\
\n\
module load python/2.7.5\n\
module remove xerces/2.7\n\
module load xerces/2.8\n\
module load gdal/1.11.0-py2.7\n\
\n\
FileConfig=%s\n\
export ITK_AUTOLOAD_PATH=""\n\
export OTB_HOME=$(grep --only-matching --perl-regex "^((?!#).)*(?<=OTB_HOME\:).*" $FileConfig | cut -d "\'" -f 2)\n\
. $OTB_HOME/config_otb.sh\n\
export ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS=1\n\
\n\
PYPATH=$(grep --only-matching --perl-regex "^((?!#).)*(?<=pyAppPath\:).*" $FileConfig | cut -d "\'" -f 2)\n\
Nsample=$(grep --only-matching --perl-regex "^((?!#).)*(?<=runs\:).*" $FileConfig | cut -d "\'" -f 2)\n\
cd $PYPATH\n\
\n\
ListeTuile=($(grep --only-matching --perl-regex "^((?!#).)*(?<=listTile\:).*" $FileConfig | cut -d "\'" -f 2))\n\
\n\
python outStats.py -tile ${ListeTuile[0]} -conf $FileConfig --sample $Nsample --wd $TMPDIR'
                      % (logPath, logPath, pathConf))
        jobFile.close()
示例#33
0
 def __init__(self):
     super(GetLoserBoardCommand, self)
     config = Config()
     config.connect_to_db()
示例#34
0
    for line in open(os.path.join(model_path, 'top_keywords')):
        line = line.strip()
        items = line.split('\t')
        label = int(items[0])
        words = items[1]
        label2topn[label] = words
    return label2topn


print('init config params')
args = vars()
args['mode'] = 'infer'
args['model_dir'] = 'service_model'
args['beam_width'] = 5
args['length_penalty_weight'] = 0.8
config = Config(**args)

stop_words = load_stop_words(config.model_dir)
word2label = load_keyword_label(config.model_dir)
label2topn = load_top_words(config.model_dir)

model = model_factory.create_model(config)

infer_model = model.create_infer_model_graph()
config_proto = model_helper.get_config_proto(config.log_device)

sess = tf.InteractiveSession(graph=infer_model.graph, config=config_proto)

ckpt = tf.train.latest_checkpoint(config.model_dir)
loaded_infer_model = model_helper.load_model(infer_model.model, ckpt, sess, "infer")
示例#35
0
                      type="str",
                      default='./config.yaml',
                      dest="config")
    parser.add_option('-m',
                      '--model_path',
                      type="str",
                      default='model',
                      dest="model_path")
    parser.add_option('-g',
                      '--graph_path',
                      type="str",
                      default='graph',
                      dest="graph_path")
    opt, args = parser.parse_args()
    server_port = opt.port
    conf_path = opt.config
    model_path = opt.model_path
    graph_path = opt.graph_path
    system_config = Config(conf_path=conf_path,
                           model_path=model_path,
                           graph_path=graph_path)
    interface_manager = InterfaceManager()
    threading.Thread(target=event_loop).start()

    logger = system_config.logger
    server_host = "0.0.0.0"

    logger.info('Running on http://{}:{}/ <Press CTRL + C to quit>'.format(
        server_host, server_port))
    serve()
示例#36
0
 def testprep_confobj(self):
     conf = Config()
     conf.setdefault()
     conf.setqualified(["A", "B", "C", "D", "E", "F", "G"])
     conf.setregular(["a", "b", "c", "d", "e", "f", "g", "h", "i"])
     conf.setmonth(1)
     conf.setyear(2001)
     restr = re.Restrictions()
     restr._setall([3, 1, 10, 1, 0, 1, 6, 3, 7, 1, 2, 3])
     conf.setrestrictions(restr.dict)
     conf.setlanguage('en')
     return conf
from data_tools.target_extractors import create_corrupted_dep_vocab, create_struct_gold_distances
from data_tools.data_inits import parse_all_corpora
from runners.trainers import train_dep_parsing
from data_tools.target_extractors import create_corrupted_dep_vocab, create_struct_gold_distances

# %%
# Custom Configuration|
from config import Config

sample_config: Config = Config(
    run_label='sample_data',
    uses_sample=True,
    path_to_data_train='data/sample/en_ewt-ud-train.conllu',
    path_to_data_valid='data/sample/en_ewt-ud-dev.conllu',
    feature_model_type=ARGS.feature_model_type or 'distilgpt2',
    will_train_simple_probe=True,
    will_control_task_simple_prob=True,
    will_train_structural_probe=True,
    will_train_dependency_probe=False,
    will_control_task_dependency_probe=False,
    struct_probe_train_epoch=100,
    dep_probe_train_epoch=1000,
    struct_probe_lr=0.001)

full_config: Config = Config(run_label='full_run_monday',
                             uses_sample=False,
                             path_to_data_train='data/en_ewt-ud-train.conllu',
                             path_to_data_valid='data/en_ewt-ud-dev.conllu',
                             feature_model_type=ARGS.feature_model_type
                             or 'xlm-roberta-base',
                             will_train_simple_probe=True,
                             will_control_task_simple_prob=True,
示例#38
0
def main():
    # configuration
    config = Config()
    config.parse_arg(FLAGS)
    config.setup_path()
    config.print_arg()

    # dataset
    if (config.dataset == 'wikibio'):
        dset = DatasetTable2text(config)
        dset.load()
    else:
        dset = Dataset(config)
        dset.build()
    config.vocab_size = len(dset.word2id)
    config.key_size = len(dset.key2id)
    config.dec_start_id = dset.word2id["_GOO"]
    config.dec_end_id = dset.word2id["_EOS"]
    config.pad_id = dset.pad_id
    config.stop_words = dset.stop_words

    # model
    if (config.model_name == "seq2seq"):
        if (config.dataset == 'wikibio'): Model = Seq2seqData2text
        else: Model = Seq2seq
    elif (config.model_name == "bow_seq2seq"): Model = BowSeq2seq
    elif (config.model_name == "vae"): Model = Vae
    elif (config.model_name == "hierarchical_vae"): Model = Hierarchical_Vae
    elif (config.model_name == "latent_bow"):
        if (config.dataset == 'wikibio'): Model = LatentBowData2text
        else: Model = LatentBow
    elif (config.model_name == "lm"): Model = LM
    else:
        msg = "the model name shoule be in ['seq2seq', 'vae', 'hierarchical_vae', 'latent_low', 'lm'], "
        msg += "current name: %s" % config.model_name
        raise Exception(msg)

    model = Model(config)
    with tf.variable_scope(config.model_name):
        model.build()

    # controller
    controller = Controller(config)
    if (config.model_name != "lm"):
        if ("lm" in controller.eval_metrics_list):
            controller.build_lm(LM, config)
    controller.train(model, dset)
    return
示例#39
0
from config import Config

from aucReader import Dataloader

test = 0
user = 0
als = 0
attention = 1
config = Config()
if test:
    config.data_path = '../data/ml-1m/sample.csv'
    config.num_users = 50
    config.train_path = '../data/ml-1m/train2'

dl = Dataloader(config)
if user:
    from userRnn import UserRNN
    print('user model test:%d' % test)
    model = UserRNN(config, dl)
elif attention:

    from attURnn import AttUserRNN
    print('attention model test:%d' % test)
    model = AttUserRNN(config, dl)
elif als:
    from als import ALS
    print('als model test:%d' % test)
    model = ALS(config, dl)
else:
    from rnnRS import RnnRs
    print('rnn model test:%d' % test)
示例#40
0
文件: logger.py 项目: SimonBorin/yoba
 def __init__(self):
     from config import Config
     from odr.container import register
     config = Config()
     register(config)
     self.level = config.log_level
示例#41
0
                label_token_list = []
                tokens_list = []

    # for index, token_list in enumerate(all_tokens_list[:146]):
    #     label_token_list = all_label_token_list[index]
    #     with open(data_dir + 'test.txt', 'a', encoding='utf-8') as fw:
    #         for i, token in enumerate(token_list):
    #             label_token = label_token_list[i]
    #             fw.write(token + ' ' + label_token + '\n')
    #         fw.write('\n')

    label_list = extract_entity(all_label_token_list, all_tokens_list)
    return text_list, label_list

if __name__ == '__main__':
    config = Config()
    data_dir = config.processed_data
    text_l, label_l = get_clean_csv(data_dir + 'source_train.txt')
    dev_text_l, dev_label_l = get_clean_csv(data_dir + 'dev.txt')

    text_l.extend(dev_text_l)
    label_l.extend(dev_label_l)
    final_label_list = []
    for l_l in label_l:
        temp_list = []
        for key in l_l:
            temp_list.extend(l_l[key])
        final_label_list.append(";".join(temp_list))

    data_dict = {'text': text_l, 'unknownEntities': final_label_list}
    data_df = pd.DataFrame(data_dict)
示例#42
0
def worker(action, dry):
    try:
        instance = input("Choose an instance tag to act upon(config.ini):")
        instance_id = Config.get_from_section(instance, 'id')

        if action.lower() == "on":
            print("You are starting an instance")
            try:
                response = ec2.start_instances(InstanceIds=[instance_id],
                                               DryRun=dry)
                print(response)
                update(instance, 'state', 'running')
                print(
                    "Please remember to turn off your instance when you're done."
                )
            except ClientError as e:
                if 'DryRunOperation' not in str(e):
                    print(permission)
                    raise
                print(error)

        elif action.lower() == "off":
            print("You are stopping an instance")
            try:
                response = ec2.stop_instances(InstanceIds=[instance_id],
                                              DryRun=dry)
                print(response)
                update(instance, 'state', 'stopped')
            except ClientError as e:
                if 'DryRunOperation' not in str(e):
                    print(permission)
                    raise
                print(error)

        elif action.lower() == "reboot":
            print("You are rebooting an instance")
            try:
                response = ec2.reboot_instances(InstanceIds=[instance_id],
                                                DryRun=dry)
                print(response)
            except ClientError as e:
                if 'DryRunOperation' not in str(e):
                    print(e)
                    raise
                print(error)

        elif action.lower() == "terminate":
            print("You are terminating an instance")
            try:
                response = ec2.terminate_instances(InstanceIds=[instance_id],
                                                   DryRun=dry)
                print(response)
                update(instance, 'state', 'terminated')
            except ClientError as e:
                if 'DryRunOperation' not in str(e):
                    print(error)
                    raise

        else:
            print("Please enter a valid action. Look at help for instructions")
    except ClientError as e:
        print(e)
    except ParamValidationError as e:
        print("Please enter arguments and values")
示例#43
0
from mpl_toolkits.mplot3d import Axes3D  # noqa: F401 unused import
import numpy as np
np.set_printoptions(threshold=np.nan)
import os, shutil
import glob
import platform
import copy
import time

from utils import read, compute_voxelgrid_and_sceneflow, generate_numpy, create_dir
from PlyFile import PlyFile
from config import Config, DATA_TYPES_3D

##################################################################################
system = platform.system()
cfg = Config(system=system)
print("Config init from ", cfg.dataset_path)
##################################################################################

# generate = input("Generate numpy files? ")
# if generate == "Y" or generate == "y":
#     generate = True
# elif generate == "F" or generate == "f":
#     generate = False
#
# print(generate)

SPLIT = "TRAIN"
LETTER = "B"
NUMBER = "0740"
示例#44
0
class BiRNN(object):
    def __init__(self, wav_files, text_labels, words_size, words,
                 word_num_map):
        self.conf = Config()
        self.wav_files = wav_files
        self.text_labels = text_labels
        self.words_size = words_size
        self.words = words
        self.word_num_map = word_num_map
        tf.logging.info('字表大小:' + str(self.words_size))

    def add_placeholders(self):
        # the batch_size and max_stepsize每步都是变长的。
        self.input_tensor = tf.placeholder(
            tf.float32, [None, None, n_input + (2 * n_input * n_context)],
            name='input')  # 语音log filter bank or MFCC features

        self.text = tf.sparse_placeholder(tf.int32, name='text')  # 文本
        self.seq_length = tf.placeholder(tf.int32, [None],
                                         name='seq_length')  # 序列长
        self.keep_dropout = tf.placeholder(tf.float32)

    def bi_rnn_layer(self):
        '''
        建立网络模型
        :param batch_x:
        :param seq_length:
        :param n_input:
        :param n_context:
        :param n_character:
        :param keep_dropout:
        '''
        batch_x = self.input_tensor
        seq_length = self.seq_length
        n_character = self.words_size + 1 + 1
        keep_dropout = self.keep_dropout

        # batch_x_shape: [batch_size, n_steps, n_input + 2*n_input*n_context]
        batch_x_shape = tf.shape(batch_x)

        # 将输入转成时间序列优先
        batch_x = tf.transpose(batch_x, [1, 0, 2])
        # 再转成2维传入第一层
        batch_x = tf.reshape(
            batch_x, [-1, n_input + 2 * n_input * n_context
                      ])  # (n_steps*batch_size, n_input + 2*n_input*n_context)

        # 1st layer
        with tf.name_scope('layer1'):
            b1 = self.variable_on_device(
                'b1', [n_hidden_1],
                tf.random_normal_initializer(stddev=b_stddev))
            h1 = self.variable_on_device(
                'h1', [n_input + 2 * n_input * n_context, n_hidden_1],
                tf.random_normal_initializer(stddev=h_stddev))
            layer_1 = tf.minimum(
                tf.nn.relu(tf.add(tf.matmul(batch_x, h1), b1)), relu_clip)
            layer_1 = tf.nn.dropout(layer_1, keep_dropout)

        # 2nd layer
        with tf.name_scope('layer2'):
            b2 = self.variable_on_device(
                'b2', [n_hidden_2],
                tf.random_normal_initializer(stddev=b_stddev))
            h2 = self.variable_on_device(
                'h2', [n_hidden_1, n_hidden_2],
                tf.random_normal_initializer(stddev=h_stddev))
            layer_2 = tf.minimum(
                tf.nn.relu(tf.add(tf.matmul(layer_1, h2), b2)), relu_clip)
            layer_2 = tf.nn.dropout(layer_2, keep_dropout)

        # 3rd layer
        with tf.name_scope('layer3'):
            b3 = self.variable_on_device(
                'b3', [n_hidden_3],
                tf.random_normal_initializer(stddev=b_stddev))
            h3 = self.variable_on_device(
                'h3', [n_hidden_2, n_hidden_3],
                tf.random_normal_initializer(stddev=h_stddev))
            layer_3 = tf.minimum(
                tf.nn.relu(tf.add(tf.matmul(layer_2, h3), b3)), relu_clip)
            layer_3 = tf.nn.dropout(layer_3, keep_dropout)

        # 双向rnn
        with tf.name_scope('birnn'):
            # 前向
            lstm_fw_cell = tf.contrib.rnn.BasicLSTMCell(n_cell_dim,
                                                        forget_bias=1.0,
                                                        state_is_tuple=True)
            lstm_fw_cell = tf.contrib.rnn.DropoutWrapper(
                lstm_fw_cell, input_keep_prob=keep_dropout)
            # 后向
            lstm_bw_cell = tf.contrib.rnn.BasicLSTMCell(n_cell_dim,
                                                        forget_bias=1.0,
                                                        state_is_tuple=True)
            lstm_bw_cell = tf.contrib.rnn.DropoutWrapper(
                lstm_bw_cell, input_keep_prob=keep_dropout)

            # `layer_3`  `[n_steps, batch_size, 2*n_cell_dim]`
            layer_3 = tf.reshape(layer_3, [-1, batch_x_shape[0], n_hidden_3])

            outputs, output_states = tf.nn.bidirectional_dynamic_rnn(
                cell_fw=lstm_fw_cell,
                cell_bw=lstm_bw_cell,
                inputs=layer_3,
                dtype=tf.float32,
                time_major=True,
                sequence_length=seq_length)

            # 连接正反向结果[n_steps, batch_size, 2*n_cell_dim]
            outputs = tf.concat(outputs, 2)
            # to a single tensor of shape [n_steps*batch_size, 2*n_cell_dim]
            outputs = tf.reshape(outputs, [-1, 2 * n_cell_dim])

        with tf.name_scope('layer5'):
            b5 = self.variable_on_device(
                'b5', [n_hidden_5],
                tf.random_normal_initializer(stddev=b_stddev))
            h5 = self.variable_on_device(
                'h5', [(2 * n_cell_dim), n_hidden_5],
                tf.random_normal_initializer(stddev=h_stddev))
            layer_5 = tf.minimum(
                tf.nn.relu(tf.add(tf.matmul(outputs, h5), b5)), relu_clip)
            layer_5 = tf.nn.dropout(layer_5, keep_dropout)

        with tf.name_scope('layer6'):
            # 全连接层用于softmax分类
            b6 = self.variable_on_device(
                'b6', [n_character],
                tf.random_normal_initializer(stddev=b_stddev))
            h6 = self.variable_on_device(
                'h6', [n_hidden_5, n_character],
                tf.random_normal_initializer(stddev=h_stddev))
            layer_6 = tf.add(tf.matmul(layer_5, h6), b6)

        # 将2维[n_steps*batch_size, n_character]转成3维 time-major [n_steps, batch_size, n_character].
        layer_6 = tf.reshape(layer_6, [-1, batch_x_shape[0], n_character])

        # Output shape: [n_steps, batch_size, n_character]
        self.logits = layer_6

    def loss(self):
        """
        定义loss
        :return:
        """
        # 调用ctc loss
        with tf.name_scope('loss'):  #损失
            self.avg_loss = tf.reduce_mean(
                ctc_ops.ctc_loss(self.text, self.logits, self.seq_length))
            tf.summary.scalar('loss', self.avg_loss)
        # [optimizer]
        with tf.name_scope('train'):  #训练过程
            self.optimizer = tf.train.AdamOptimizer(
                learning_rate=learning_rate).minimize(self.avg_loss)

        with tf.name_scope("decode"):
            self.decoded, log_prob = ctc_ops.ctc_beam_search_decoder(
                self.logits,
                self.seq_length,
                beam_width=10,
                merge_repeated=True)

        with tf.name_scope("accuracy"):
            self.distance = tf.edit_distance(
                tf.cast(self.decoded[0], tf.int32), self.text)
            # 计算label error rate (accuracy)
            self.label_err = tf.reduce_mean(self.distance,
                                            name='label_error_rate')
            tf.summary.scalar('accuracy', self.label_err)

    def get_feed_dict(self, dropout=None):
        """
        定义变量
        :param dropout:
        :return:
        """
        feed_dict = {
            self.input_tensor: self.audio_features,
            self.text: self.sparse_labels,
            self.seq_length: self.audio_features_len
        }

        if dropout != None:
            feed_dict[self.keep_dropout] = dropout
        else:
            feed_dict[self.keep_dropout] = keep_dropout_rate

        return feed_dict

    def init_session(self):
        self.savedir = self.conf.get("FILE_DATA").savedir
        self.saver = tf.train.Saver(max_to_keep=1)  # 生成saver
        # create the session
        gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.7)
        self.sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
        # sess = tf.Session()
        # 没有模型的话,就重新初始化
        self.sess.run(tf.global_variables_initializer())

        ckpt = tf.train.latest_checkpoint(self.savedir)
        tf.logging.info("ckpt:" + str(ckpt))
        self.startepo = 0
        if ckpt != None:
            self.saver.restore(self.sess, ckpt)
            ind = ckpt.rfind("-")
            self.startepo = int(ckpt[ind + 1:])
            tf.logging.info(str(self.startepo))

    def add_summary(self):
        self.merged = tf.summary.merge_all()
        self.writer = tf.summary.FileWriter(
            self.conf.get("FILE_DATA").tensorboardfile, self.sess.graph)

    def train(self):
        epochs = 200

        # 准备运行训练步骤
        section = '\n{0:=^40}\n'
        tf.logging.info(section.format('开始训练'))

        train_start = time.time()
        for epoch in range(epochs):  # 样本集迭代次数
            epoch_start = time.time()
            if epoch < self.startepo:
                continue

            tf.logging.info("第" + str(epoch + 1) + "次迭代,一共要迭代" + str(epochs) +
                            "次")
            #######################run batch####
            n_batches_epoch = int(np.ceil(len(self.text_labels) / batch_size))
            tf.logging.info("在本次迭代中一共循环" + str(n_batches_epoch) + ",每次取" +
                            str(batch_size))

            train_cost = 0
            train_err = 0
            next_idx = 0

            for batch in range(n_batches_epoch):  # 一次batch_size,取多少次
                # 取数据
                # temp_next_idx, temp_audio_features, temp_audio_features_len, temp_sparse_labels
                tf.logging.info(
                    '%d/%d:%s', batch + 1, n_batches_epoch,
                    datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
                next_idx, self.audio_features, self.audio_features_len, self.sparse_labels, wav_files = utils.next_batch(
                    next_idx,
                    batch_size,
                    n_input,
                    n_context,
                    self.text_labels,
                    self.wav_files,
                    self.word_num_map,
                    pick_deterministically=False)

                # 计算 avg_loss optimizer ;
                batch_cost, _ = self.sess.run([self.avg_loss, self.optimizer],
                                              feed_dict=self.get_feed_dict())
                train_cost += batch_cost

                if (batch + 1) % 100 == 0:
                    #     rs = self.sess.run(self.merged, feed_dict=self.get_feed_dict())
                    #     self.writer.add_summary(rs, batch)

                    tf.logging.info('循环次数:' + str(batch + 1) + '损失:' +
                                    str(train_cost / (batch + 1)))

                    d, train_err = self.sess.run(
                        [self.decoded[0], self.label_err],
                        feed_dict=self.get_feed_dict(dropout=1.0))
                    dense_decoded = tf.sparse_tensor_to_dense(
                        d, default_value=-1).eval(session=self.sess)
                    dense_labels = utils.trans_tuple_to_texts_ch(
                        self.sparse_labels, self.words)

                    tf.logging.info('错误率:' + str(train_err))
                    for orig, decoded_array in zip(dense_labels,
                                                   dense_decoded):
                        # convert to strings
                        decoded_str = utils.trans_array_to_text_ch(
                            decoded_array, self.words)
                        tf.logging.info('语音原始文本:{}'.format(
                            orig.encode('utf-8')))
                        tf.logging.info('识别出来的文本:{}'.format(
                            decoded_str.encode('utf-8')))
                        break

            epoch_duration = time.time() - epoch_start

            log = '迭代次数 {}/{}, 训练损失:{:.3f}, 错误率:{:.3f}, time:{:.2f} sec'
            tf.logging.info(
                log.format(epoch + 1, epochs, train_cost, train_err,
                           epoch_duration))
            self.saver.save(self.sess,
                            self.savedir + self.conf.get("FILE_DATA").savefile,
                            global_step=epoch + 1)

        train_duration = time.time() - train_start
        tf.logging.info('Training complete, total duration:{:.2f} min'.format(
            train_duration / 60))
        self.sess.close()

    def test(self):
        index = 0
        next_idx = 20

        for index in range(10):
            next_idx, self.audio_features, self.audio_features_len, self.sparse_labels, wav_files = utils.next_batch(
                next_idx,
                1,
                n_input,
                n_context,
                self.text_labels,
                self.wav_files,
                self.word_num_map,
                pick_deterministically=True)

            tf.logging.info('读入语音文件:' + wav_files[0].encode('utf-8'))
            tf.logging.info('开始识别语音数据......')

            d, train_ler = self.sess.run(
                [self.decoded[0], self.label_err],
                feed_dict=self.get_feed_dict(dropout=1.0))
            dense_decoded = tf.sparse_tensor_to_dense(
                d, default_value=-1).eval(session=self.sess)
            dense_labels = utils.trans_tuple_to_texts_ch(
                self.sparse_labels, self.words)

            for orig, decoded_array in zip(dense_labels, dense_decoded):
                # 转成string
                decoded_str = utils.trans_array_to_text_ch(
                    decoded_array, self.words)
                tf.logging.info('语音原始文本:{}'.format(orig.encode('utf-8')))
                tf.logging.info('识别出来的文本:{}'.format(
                    decoded_str.encode('utf-8')))
                break

        self.sess.close()

    def test_target_wav_file(self, wav_files, txt_labels):
        tf.logging.info('读入语音文件:' + wav_files[0].encode('utf-8'))
        tf.logging.info('开始识别语音数据......')

        self.audio_features, self.audio_features_len, text_vector, text_vector_len = utils.get_audio_mfcc_features(
            None, wav_files, n_input, n_context, self.word_num_map, txt_labels)
        self.sparse_labels = utils.sparse_tuple_from(text_vector)
        d, train_ler = self.sess.run([self.decoded[0], self.label_err],
                                     feed_dict=self.get_feed_dict(dropout=1.0))
        dense_decoded = tf.sparse_tensor_to_dense(
            d, default_value=-1).eval(session=self.sess)
        decoded_str = utils.trans_array_to_text_ch(dense_decoded[0],
                                                   self.words)
        tf.logging.info('语音原始文本:{}'.format(txt_labels[0].encode('utf-8')))
        tf.logging.info('识别出来的文本:{}'.format(decoded_str.encode('utf-8')))

        self.sess.close()

    def build_train(self):
        self.add_placeholders()
        self.bi_rnn_layer()
        self.loss()
        self.init_session()
        self.add_summary()
        self.train()

    def build_test(self):
        self.add_placeholders()
        self.bi_rnn_layer()
        self.loss()
        self.init_session()
        self.test()

    def build_target_wav_file_test(self, wav_files, txt_labels):
        self.add_placeholders()
        self.bi_rnn_layer()
        self.loss()
        self.init_session()
        self.test_target_wav_file(wav_files, txt_labels)

    def variable_on_device(self, name, shape, initializer):
        with tf.device('/gpu:0'):
            var = tf.get_variable(name=name,
                                  shape=shape,
                                  initializer=initializer)
        return var
示例#45
0
from math import pow, sqrt
from datetime import datetime as dt
from config import Config
from sheet import PostUpdate
import re

# Logger instance used by the functions in this module
_LOGGER = Config.getLogger("influence")

# Determine if only looking for events today
_TODAY_ONLY = Config.getBoolean('events', 'today_only', True)
_ROUND_DISTANCE = Config.getInteger('events', 'distancedp', -1)
# Allow specific factions to be ignored
_IGNORE_FACTION_SET = set()
_IGNORE_FACTIONS = Config.getString('events', 'ignore_factions')
if _IGNORE_FACTIONS is not None and len(_IGNORE_FACTIONS.strip()) > 0:
    _IGNORE_FACTION_SET.update([faction.strip() for faction in _IGNORE_FACTIONS.split(",")])

# Interested in activity around a specified location
_LOCATION_X = Config.getFloat('location', 'x')
_LOCATION_Y = Config.getFloat('location', 'y')
_LOCATION_Z = Config.getFloat('location', 'z')
_RANGE_SQUARED = Config.getFloat('location', 'distance')**2
_LOGGER.info("Configured for %.1f LY around %s", Config.getFloat('location', 'distance'), Config.getString('location', 'name'))

# Provide regular expressions to remove extraneous text specifiers
_MATCH_GOV = re.compile(r'\$government_(.*);', re.IGNORECASE)
_MATCH_SEC = re.compile(r'\$system_security_(.*);', re.IGNORECASE)
_MATCH_SEC2 = re.compile(r'\$GAlAXY_MAP_INFO_state_(.*);', re.IGNORECASE)
_MATCH_ECO = re.compile(r'\$economy_(.*);', re.IGNORECASE)
示例#46
0
import argparse
from bottle import request, response, route, run
import bottle
from command import Command
from config import Config
from hardware import Hardware
import json
import logging
from time import time

parser = argparse.ArgumentParser()
c = Config()

parser.add_argument('--node',
                    help='Node id',
                    default='master',
                    choices=c.settings['nodes'].keys())
parser.add_argument('--loglevel',
                    help='Log level for server & playback',
                    default='INFO')
args = parser.parse_args()

logging.getLogger().setLevel(args.loglevel)
hw = Hardware(c, args.node)

script = []


@route('/cmd', method='POST')
def start_show():
    global script
示例#47
0
"""

import json
import requests
import datetime
import argparse
import os
import time
import sys

sys.path.append(os.path.abspath('./utils'))
from config import Config
from database import Database
from ssh import SSH

nebula_config = Config(
    os.path.abspath('./guard/config/nebula_config.yml')).config

# web configurations
HTTP_HOST = nebula_config.get("http").get("host")
HTTP_PORT = nebula_config.get("http").get("port")
HTTP_END_POINT = "http://%s:%s" % (HTTP_HOST, HTTP_PORT)
HTTP_RESOURCE_AC_MANAGER_TASK_LIST = nebula_config.get("http").get(
    "resource").get("AccessControlMasterService").get("acManagerTaskList")
HTTP_RESOURCE_AC_MANAGER_TASK_DELETE = nebula_config.get("http").get(
    "resource").get("AccessControlMasterService").get("acManagerTaskDelete")
HTTP_RESOURCE_TD_MANAGER_TASK_LIST = nebula_config.get("http").get(
    "resource").get("TDComparisonService").get("tdManagerTaskList")
HTTP_RESOURCE_TD_MANAGER_TASK_DELETE = nebula_config.get("http").get(
    "resource").get("TDComparisonService").get("tdManagerTaskDelete")
HTTP_RESOURCE_VIDEO_PROCESS_TASK_LIST = nebula_config.get("http").get(
    "resource").get("VideoProcessService").get("videoProcessTaskList")
示例#48
0
                    str(water_pig_data.tag_id) + ", drink value: " + str(water_pig_data.val)
                log('info', logstr)
                print(logstr)

                water_pig_data = None
            time.sleep(0.001)

    water_rfid_thread = threading.Thread(target=water_rfid_listen)
    water_rfid_thread.start()

    water_main_thread = threading.Thread(target=water_main_listen)
    water_main_thread.start()


if __name__ == "__main__":
    config = Config.get_instance()

    food_rfid = RFID(ip=config.FOOD_RFID_IP,
                     port=config.FOOD_RFID_PORT,
                     com=config.FOOD_RFID_COM,
                     name="FOOD_RFID")
    food_device = Weight_Device(ip=config.WEIGHT_IP,
                                port=config.WEIGHT_PORT,
                                com=config.WEIGHT_COM)

    water_rfid = RFID(ip=config.WATER_RFID_IP,
                      port=config.WATER_RFID_PORT,
                      com=config.WATER_RFID_COM,
                      name="WARTER_RFID")
    water_device = Water(ip=config.WATER_IP,
                         port=config.WATER_PORT,
示例#49
0
def main():
    config = Config(DATA)
    controller = MainController(config)
    controller.run()
示例#50
0
            trunc = s[-maxlen:]
        elif truncating == 'post':
            trunc = s[:maxlen]
        else:
            raise ValueError('Truncating type "%s" not understood' %
                             truncating)

        # check `trunc` has expected shape
        trunc = np.asarray(trunc, dtype=dtype)
        if trunc.shape[1:] != sample_shape:
            raise ValueError(
                'Shape of sample %s of sequence at position %s is different from expected shape %s'
                % (trunc.shape[1:], idx, sample_shape))

        if padding == 'post':
            x[idx, :len(trunc)] = trunc
        elif padding == 'pre':
            x[idx, -len(trunc):] = trunc
        else:
            raise ValueError('Padding type "%s" not understood' % padding)
    return x, sequences_each_len


if __name__ == "__main__":
    conf = Config()

    get_wavs_lables(
        conf.get("FILE_DATA").wav_path,
        conf.get("FILE_DATA").label_file)
    print()
示例#51
0
"""
利用resnet50提取图片的语义信息
并保存层results.pth
"""
from config import Config
import tqdm
import torch as t
from torch.autograd import Variable
import torchvision as tv
from torch.utils import data
import os
from PIL import Image
import numpy as np

t.set_grad_enabled(False)
opt = Config()

IMAGENET_MEAN = [0.485, 0.456, 0.406]
IMAGENET_STD = [0.229, 0.224, 0.225]
normalize = tv.transforms.Normalize(mean=IMAGENET_MEAN, std=IMAGENET_STD)


class CaptionDataset(data.Dataset):
    def __init__(self, caption_data_path):
        self.transforms = tv.transforms.Compose([
            tv.transforms.Resize(256),
            tv.transforms.CenterCrop(256),
            tv.transforms.ToTensor(), normalize
        ])

        data = t.load(caption_data_path)
示例#52
0
import os
import subprocess
import sys
import tempfile
import time

from fabric.api import local, env, run, sudo

from config import Config

# environment variable prefix
prefix = Config.environment_variable_prefix()

# disable logging (as otherwise we would have to use the production setting for the log file location)
os.environ[prefix + 'WITH_LOGGING'] = '0'

# get variables
settings = Config.settings('production')
host = os.environ[prefix + 'DEPLOY_HOST']
deploy_user = os.environ.get(prefix + 'DEPLOY_USER', 'deploy')
deploy_user_group = os.environ.get(prefix + 'DEPLOY_USER_GROUP', deploy_user)
repository = os.environ[prefix + 'DEPLOY_GIT_REPOSITORY']
app_dir_name = os.environ[prefix + 'DEPLOY_APP_DIR_NAME']
web_user = os.environ.get(prefix + 'DEPLOY_WEB_USER', 'www-data')
web_user_group = os.environ.get(prefix + 'DEPLOY_WEB_USER_GROUP', 'www-data')
domain_name = os.environ.get(prefix + 'DEPLOY_DOMAIN_NAME', host)
bokeh_server_port = os.environ.get(prefix + 'DEPLOY_BOKEH_SERVER_PORT', 5100)
migration_tool = settings['migration_tool']
migration_sql_dir = settings['migration_sql_dir']

site_dir = '$HOME/' + app_dir_name
示例#53
0
    # without data, or later with data.
    curprep.execute(prepstmt)

    # Insert 3 records
    names = ('Geert', 'Jan', 'Michel')
    for name in names:
        curprep.execute(prepstmt, (name, ))
        cnx.commit()

    # We use a normal cursor issue a SELECT
    output.append("Inserted data")
    cur.execute("SELECT id, name FROM names")
    for row in cur:
        output.append("%d | %s" % (row[0], row[1]))

    # Cleaning up, dropping the table again
    cur.execute(stmt_drop)

    cnx.close()
    return output


if __name__ == '__main__':
    #
    # Configure MySQL login and database to use in config.py
    #
    from config import Config
    config = Config.dbinfo().copy()
    out = main(config)
    print('\n'.join(out))
示例#54
0
def launchClassification(tempFolderSerie, Classifmask, model, stats,
                         outputClassif, confmap, pathWd, pathConf, pixType):
    outputClassif = outputClassif.replace(".tif", "_TMP.tif")
    confmap = confmap.replace(".tif", "_TMP.tif")
    os.environ["ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS"] = "5"
    featuresPath = Config(file(pathConf)).chain.featuresPath
    outputPath = Config(file(pathConf)).chain.outputPath
    tile = outputClassif.split("/")[-1].split("_")[1]
    userFeatPath = Config(file(pathConf)).chain.userFeatPath
    if userFeatPath == "None": userFeatPath = None
    AllRefl = sorted(
        fu.FileSearch_AND(featuresPath + "/" + tile + "/tmp/", True,
                          "REFL.tif"))
    AllMask = sorted(
        fu.FileSearch_AND(featuresPath + "/" + tile + "/tmp/", True,
                          "MASK.tif"))
    datesInterp = sorted(
        fu.FileSearch_AND(featuresPath + "/" + tile + "/tmp/", True,
                          "DatesInterp"))
    realDates = sorted(
        fu.FileSearch_AND(featuresPath + "/" + tile + "/tmp/", True,
                          "imagesDate"))

    tmpFolder = outputPath + "/TMPFOLDER_" + tile
    if not os.path.exists(tmpFolder):
        os.mkdir(tmpFolder)
        # Sensors
    S2 = Sensors.Sentinel_2("", Opath(tmpFolder), pathConf, "")
    L8 = Sensors.Landsat8("", Opath(tmpFolder), pathConf, "")
    L5 = Sensors.Landsat5("", Opath(tmpFolder), pathConf, "")
    # shutil.rmtree(tmpFolder, ignore_errors=True)

    SensorsList = [S2, L8, L5]
    # gapFill + feat
    features = []
    concatSensors = otb.Registry.CreateApplication("ConcatenateImages")
    for refl, mask, datesInterp, realDates in zip(AllRefl, AllMask,
                                                  datesInterp, realDates):
        gapFill = otb.Registry.CreateApplication("ImageTimeSeriesGapFilling")
        nbDate = fu.getNbDateInTile(realDates)
        nbReflBands = fu.getRasterNbands(refl)
        comp = int(nbReflBands) / int(nbDate)
        if not isinstance(comp, int):
            raise Exception("unvalid component by date (not integer) : " +
                            comp)
        gapFill.SetParameterString("in", refl)
        gapFill.SetParameterString("mask", mask)
        gapFill.SetParameterString("comp", str(comp))
        gapFill.SetParameterString("it", "linear")
        gapFill.SetParameterString("id", realDates)
        gapFill.SetParameterString("od", datesInterp)
        # gapFill.SetParameterString("ram","1024")
        gapFill.Execute()

        # featExtr = otb.Registry.CreateApplication("iota2FeatureExtraction")
        # featExtr.SetParameterInputImage("in",gapFill.GetParameterOutputImage("out"))
        # featExtr.SetParameterString("comp",str(comp))
        # for currentSensor in SensorsList:
        #    if currentSensor.name in refl:
        #	    red = str(currentSensor.bands["BANDS"]["red"])
        #	    nir = str(currentSensor.bands["BANDS"]["NIR"])
        #	    swir = str(currentSensor.bands["BANDS"]["SWIR"])
        # featExtr.SetParameterString("red",red)
        # featExtr.SetParameterString("nir",nir)
        # featExtr.SetParameterString("swir",swir)
        # featExtr.Execute()

        # features.append(featExtr)
        concatSensors.AddImageToParameterInputImageList(
            "il", gapFill.GetParameterOutputImage("out"))
        features.append(gapFill)

    classifier = otb.Registry.CreateApplication("ImageClassifier")
    classifier.SetParameterString("mask", Classifmask)
    if stats: classifier.SetParameterString("imstat", stats)
    classifier.SetParameterString("out", outputClassif)
    classifier.SetParameterString("model", model)
    classifier.SetParameterString("confmap", confmap)
    classifier.SetParameterString("ram", "512")
    print "AllRefl"
    print AllRefl
    # if len(AllRefl) >1:
    #	concatSensors.Execute()
    #	classifier.SetParameterInputImage("in",concatSensors.GetParameterOutputImage("out"))
    # else:
    #	classifier.SetParameterInputImage("in",features[0].GetParameterOutputImage("out"))
    # classifier.ExecuteAndWriteOutput()
    if len(AllRefl) > 1:
        concatSensors.Execute()
        allFeatures = concatSensors.GetParameterOutputImage("out")
    else:
        allFeatures = features[0].GetParameterOutputImage("out")

    if userFeatPath:
        print "Add user features"
        userFeat_arbo = Config(file(pathConf)).userFeat.arbo
        userFeat_pattern = (Config(
            file(pathConf)).userFeat.patterns).split(",")
        concatFeatures = otb.Registry.CreateApplication("ConcatenateImages")
        userFeatures = fu.getUserFeatInTile(userFeatPath, tile, userFeat_arbo,
                                            userFeat_pattern)
        concatFeatures.SetParameterStringList("il", userFeatures)
        concatFeatures.Execute()

        concatAllFeatures = otb.Registry.CreateApplication("ConcatenateImages")
        concatAllFeatures.AddImageToParameterInputImageList("il", allFeatures)
        concatAllFeatures.AddImageToParameterInputImageList(
            "il", concatFeatures.GetParameterOutputImage("out"))
        concatAllFeatures.Execute()

        allFeatures = concatAllFeatures.GetParameterOutputImage("out")

    classifier.SetParameterInputImage("in", allFeatures)
    classifier.ExecuteAndWriteOutput()

    expr = "im2b1>=1?im1b1:0"
    cmd = 'otbcli_BandMath -il ' + outputClassif + ' ' + Classifmask + ' -out ' + outputClassif.replace(
        "_TMP.tif", ".tif") + ' -exp "' + expr + '"'
    print cmd
    os.system(cmd)

    cmd = 'otbcli_BandMath -il ' + confmap + ' ' + Classifmask + ' -out ' + confmap.replace(
        "_TMP.tif", ".tif") + ' -exp "' + expr + '"'
    print cmd
    os.system(cmd)

    if pathWd:
        shutil.copy(outputClassif.replace("_TMP.tif", ".tif"),
                    outputPath + "/classif")
    if pathWd:
        shutil.copy(confmap.replace("_TMP.tif", ".tif"),
                    outputPath + "/classif")
    os.remove(outputClassif)
    os.remove(confmap)
示例#55
0
 def __init__(self, cfg_path=None):
     self._cfg = Config(cfg_path or get_default_str_cfg_path())
     self._init_datasets()
示例#56
0
文件: adresar.py 项目: mikicz/flexipy
 def __init__(self, conf=Config()):
     Flexipy.__init__(self, config=conf)
示例#57
0
from flask import Flask
from config import Config
from flask_sqlalchemy import SQLAlchemy
from flask_migrate import Migrate
from flask_login import LoginManager
from flask_bootstrap import Bootstrap

import logging
from logging.handlers import RotatingFileHandler
import os

app = Flask(__name__)
conf = Config()
app.config.from_object(conf)
bootstrap = Bootstrap(app)

login = LoginManager(app)
login.login_view = 'login'

db = SQLAlchemy(app)
migrate = Migrate(app, db)

from app import routes, models, errors

if not app.debug:
    if not os.path.exists('logs'):
        os.mkdir('logs')
    file_handler = RotatingFileHandler('logs/RPGforumCMS.log',
                                       maxBytes=10240,
                                       backupCount=10)
    file_handler.setFormatter(
示例#58
0
from detect_ball import DetectBall
from config import Config
from draw import Draw
from preprocess import simplest_cb
from db_master import DbMaster
from enum import Enum
import imutils


class VideoType(Enum):
    FILE = 0  #read video from file
    RTSP = 1  #read video from rtsp link


config = Config()
videoType = VideoType(config.get('video_type'))

cap = cv.VideoCapture()
if videoType == VideoType.FILE:
    cap = cv.VideoCapture('./data/' + config.get('video_name'))
elif videoType == VideoType.RTSP:
    cap = cv.VideoCapture(config.get('rtsp_stream'))

if __name__ == '__main__':

    master_db = DbMaster(config.get('db_name'))

    if not master_db.table_exists(config.get('table_name')):
        master_db.create_table(
            config.get('table_name'), {
示例#59
0
# TODO: try largest lr before diverging
# TODO: check all plots rendered
# TODO: adamw

FOLDS = list(range(1, 5 + 1))
DEVICE = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')

parser = argparse.ArgumentParser()
parser.add_argument('--config-path', type=str, required=True)
parser.add_argument('--experiment-path', type=str, default='./tf_log/frees')
parser.add_argument('--dataset-path', type=str, required=True)
parser.add_argument('--workers', type=int, default=os.cpu_count())
parser.add_argument('--fold', type=int, choices=FOLDS)
parser.add_argument('--debug', action='store_true')
args = parser.parse_args()
config = Config.from_yaml(args.config_path)
shutil.copy(args.config_path, utils.mkdir(args.experiment_path))

if config.aug.effects:
    extra_augs = [AudioEffect()]
else:
    extra_augs = []

if config.aug.type == 'pad':
    train_transform = T.Compose([
        LoadSignal(config.model.sample_rate),
        ToTensor(),
    ])
    eval_transform = T.Compose([
        LoadSignal(config.model.sample_rate),
        ToTensor(),
示例#60
0
class Whost(SimpleGladeApp):
    def __init__(self,
                 path="gnome-connection-manager.glade",
                 root="wHost",
                 domain=DOMAIN_NAME,
                 **kwargs):
        path = os.path.join(GLADE_DIR, path)
        SimpleGladeApp.__init__(self, path, root, domain, **kwargs)

        self.treeModel = gtk.ListStore(gobject.TYPE_STRING,
                                       gobject.TYPE_STRING,
                                       gobject.TYPE_STRING,
                                       gobject.TYPE_STRING)
        self.treeTunel.set_model(self.treeModel)
        column = gtk.TreeViewColumn(_("Local"), gtk.CellRendererText(), text=0)
        self.treeTunel.append_column(column)
        column = gtk.TreeViewColumn(_("Host"), gtk.CellRendererText(), text=1)
        self.treeTunel.append_column(column)
        column = gtk.TreeViewColumn(_("Remote"),
                                    gtk.CellRendererText(),
                                    text=2)
        self.treeTunel.append_column(column)

    def new(self):
        self.config = Config()
        self.cmbGroup = self.get_widget("cmbGroup")
        self.txtName = self.get_widget("txtName")
        self.txtDescription = self.get_widget("txtDescription")
        self.txtHost = self.get_widget("txtHost")
        self.cmbType = self.get_widget("cmbType")
        self.txtUser = self.get_widget("txtUser")
        self.txtPass = self.get_widget("txtPassword")
        self.txtPrivateKey = self.get_widget("txtPrivateKey")
        self.btnBrowse = self.get_widget("btnBrowse")
        self.txtPort = self.get_widget("txtPort")
        self.cmbGroup.get_model().clear()
        for group in self.config.groups:
            self.cmbGroup.get_model().append([group])
        self.isNew = True

        self.chkDynamic = self.get_widget("chkDynamic")
        self.txtLocalPort = self.get_widget("txtLocalPort")
        self.txtRemoteHost = self.get_widget("txtRemoteHost")
        self.txtRemotePort = self.get_widget("txtRemotePort")
        self.treeTunel = self.get_widget("treeTunel")
        self.txtComamnds = self.get_widget("txtCommands")
        self.chkComamnds = self.get_widget("chkCommands")
        buf = self.txtComamnds.get_buffer()
        buf.create_tag('DELAY1',
                       style=pango.STYLE_ITALIC,
                       foreground='darkgray')
        buf.create_tag('DELAY2',
                       style=pango.STYLE_ITALIC,
                       foreground='cadetblue')
        buf.connect("changed", self.update_texttags)
        self.chkKeepAlive = self.get_widget("chkKeepAlive")
        self.txtKeepAlive = self.get_widget("txtKeepAlive")
        self.btnFColor = self.get_widget("btnFColor")
        self.btnBColor = self.get_widget("btnBColor")
        self.chkX11 = self.get_widget("chkX11")
        self.chkAgent = self.get_widget("chkAgent")
        self.chkCompression = self.get_widget("chkCompression")
        self.txtCompressionLevel = self.get_widget("txtCompressionLevel")
        self.txtExtraParams = self.get_widget("txtExtraParams")
        self.chkLogging = self.get_widget("chkLogging")
        self.cmbBackspace = self.get_widget("cmbBackspace")
        self.cmbDelete = self.get_widget("cmbDelete")
        self.cmbType.set_active(0)
        self.cmbBackspace.set_active(0)
        self.cmbDelete.set_active(0)

    def init(self, group, host=None):
        self.cmbGroup.get_child().set_text(group)
        if host == None:
            self.isNew = True
            return

        self.isNew = False
        self.oldGroup = group
        self.txtName.set_text(host.name)
        self.oldName = host.name
        self.txtDescription.set_text(host.description)
        self.txtHost.set_text(host.host)
        i = self.cmbType.get_model().get_iter_first()
        while i != None:
            if (host.type == self.cmbType.get_model()[i][0]):
                self.cmbType.set_active_iter(i)
                break
            else:
                i = self.cmbType.get_model().iter_next(i)
        self.txtUser.set_text(host.user)
        self.txtPass.set_text(host.password)
        self.txtPrivateKey.set_text(host.private_key)
        self.txtPort.set_text(host.port)
        for t in host.tunnel:
            if t != "":
                tun = t.split(":")
                tun.append(t)
                self.treeModel.append(tun)
        self.txtCommands.set_sensitive(False)
        self.chkCommands.set_active(False)
        if host.commands != '' and host.commands != None:
            self.txtCommands.get_buffer().set_text(host.commands)
            self.txtCommands.set_sensitive(True)
            self.chkCommands.set_active(True)
        use_keep_alive = host.keep_alive != '' and host.keep_alive != '0' and host.keep_alive != None
        self.txtKeepAlive.set_sensitive(use_keep_alive)
        self.chkKeepAlive.set_active(use_keep_alive)
        self.txtKeepAlive.set_text(host.keep_alive)
        if host.font_color != '' and host.font_color != None and host.back_color != '' and host.back_color != None:
            self.get_widget("chkDefaultColors").set_active(False)
            self.btnFColor.set_sensitive(True)
            self.btnBColor.set_sensitive(True)
            fcolor = host.font_color
            bcolor = host.back_color
        else:
            self.get_widget("chkDefaultColors").set_active(True)
            self.btnFColor.set_sensitive(False)
            self.btnBColor.set_sensitive(False)
            fcolor = "#FFFFFF"
            bcolor = "#000000"

        self.btnFColor.set_color(gtk.gdk.Color(fcolor))
        self.btnBColor.set_color(gtk.gdk.Color(bcolor))

        m = self.btnFColor.get_colormap()
        color = m.alloc_color("red")
        style = self.btnFColor.get_style().copy()
        style.bg[gtk.STATE_NORMAL] = color
        self.btnFColor.set_style(style)
        self.btnFColor.queue_draw()

        self.btnFColor.selected_color = fcolor
        self.btnBColor.selected_color = bcolor
        self.chkX11.set_active(host.x11)
        self.chkAgent.set_active(host.agent)
        self.chkCompression.set_active(host.compression)
        self.txtCompressionLevel.set_text(host.compressionLevel)
        self.txtExtraParams.set_text(host.extra_params)
        self.chkLogging.set_active(host.log)
        self.cmbBackspace.set_active(host.backspace_key)
        self.cmbDelete.set_active(host.delete_key)
        self.update_texttags()

    def update_texttags(self, *args):
        buf = self.txtCommands.get_buffer()
        text_iter = buf.get_start_iter()
        buf.remove_all_tags(text_iter, buf.get_end_iter())
        while True:
            found = text_iter.forward_search("##D=", 0, None)
            if not found:
                break
            start, end = found
            n = end.copy()
            end.forward_line()
            if buf.get_text(n, end).rstrip().isdigit():
                buf.apply_tag_by_name("DELAY1", start, n)
                buf.apply_tag_by_name("DELAY2", n, end)
            text_iter = end

    def on_cancelbutton1_clicked(self, widget, *args):
        self.get_widget("wHost").destroy()

    def on_okbutton1_clicked(self, widget, *args):
        group = self.cmbGroup.get_active_text().strip()
        name = self.txtName.get_text().strip()
        description = self.txtDescription.get_text().strip()
        host = self.txtHost.get_text().strip()
        ctype = self.cmbType.get_active_text().strip()
        user = self.txtUser.get_text().strip()
        password = self.txtPass.get_text().strip()
        private_key = self.txtPrivateKey.get_text().strip()
        port = self.txtPort.get_text().strip()
        buf = self.txtCommands.get_buffer()
        commands = buf.get_text(buf.get_start_iter(), buf.get_end_iter(
        )).strip() if self.chkCommands.get_active() else ""
        keepalive = self.txtKeepAlive.get_text().strip()
        if self.get_widget("chkDefaultColors").get_active():
            fcolor = ""
            bcolor = ""
        else:
            fcolor = self.btnFColor.selected_color
            bcolor = self.btnBColor.selected_color

        x11 = self.chkX11.get_active()
        agent = self.chkAgent.get_active()
        compression = self.chkCompression.get_active()
        compressionLevel = self.txtCompressionLevel.get_text().strip()
        extra_params = self.txtExtraParams.get_text()
        log = self.chkLogging.get_active()
        backspace_key = self.cmbBackspace.get_active()
        delete_key = self.cmbDelete.get_active()

        if ctype == "":
            ctype = "ssh"
        tunnel = ""

        if ctype == "ssh":
            for x in self.treeModel:
                tunnel = '%s,%s' % (x[3], tunnel)
            tunnel = tunnel[:-1]

        # Validar datos
        if group == "" or name == "" or (host == "" and ctype != 'local'):
            msgbox(_("Fields group, name and host are required"))
            return

        if not (port and port.isdigit() and 1 <= int(port) <= 65535):
            msgbox(_("Invalid port"))
            return

        host = Host(group, name, description, host, user, password,
                    private_key, port, tunnel, ctype, commands, keepalive,
                    fcolor, bcolor, x11, agent, compression, compressionLevel,
                    extra_params, log, backspace_key, delete_key)

        try:
            # Guardar
            if not group not in self.config.groups:
                self.config.groups[group] = []

            if self.isNew:
                for h in self.config.groups[group]:
                    if h.name == name:
                        msgbox("%s [%s] %s [%s]" %
                               (_("Host name"), name,
                                _("already exists for group"), group))
                        return
                # agregar host a grupo
                self.config.groups[group].append(host)
            else:
                if self.oldGroup != group:
                    # revisar que no este el nombre en el nuevo grupo
                    if not group not in self.config.groups:
                        self.config.groups[group] = [host]
                    else:
                        for h in self.config.groups[group]:
                            if h.name == name:
                                msgbox("%s [%s] %s [%s]" %
                                       (_("Host name"), name,
                                        _("already exists for group"), group))
                                return
                        self.config.groups[group].append(host)
                        for h in self.config.groups[self.oldGroup]:
                            if h.name == self.oldName:
                                self.config.groups[self.oldGroup].remove(h)
                                break
                else:
                    if self.oldName != name:
                        for h in self.config.groups[self.oldGroup]:
                            if h.name == name:
                                msgbox("%s [%s] %s [%s]" %
                                       (_("Host name"), name,
                                        _("already exists for group"), group))
                                return
                        for h in self.config.groups[self.oldGroup]:
                            if h.name == self.oldName:
                                index = self.config.groups[
                                    self.oldGroup].index(h)
                                self.config.groups[self.oldGroup][index] = host
                                break
                    else:
                        for h in self.config.groups[self.oldGroup]:
                            if h.name == self.oldName:
                                index = self.config.groups[
                                    self.oldGroup].index(h)
                                self.config.groups[self.oldGroup][index] = host
                                break
        except:
            traceback.print_exc()
            msgbox("%s [%s]" %
                   (_("Error saving host. Description"), sys.exc_info()[1]))

        self.config.writeConfig()
        self.get_widget("wHost").destroy()

    def on_cmbType_changed(self, widget, *args):
        is_local = widget.get_active_text() == "local"
        self.txtUser.set_sensitive(not is_local)
        self.txtPassword.set_sensitive(not is_local)
        self.txtPort.set_sensitive(not is_local)
        self.txtHost.set_sensitive(not is_local)
        self.txtExtraParams.set_sensitive(not is_local)

        if widget.get_active_text() == "ssh":
            self.get_widget("table2").show_all()
            self.txtKeepAlive.set_sensitive(True)
            self.chkKeepAlive.set_sensitive(True)
            self.chkX11.set_sensitive(True)
            self.chkAgent.set_sensitive(True)
            self.chkCompression.set_sensitive(True)
            self.txtCompressionLevel.set_sensitive(
                self.chkCompression.get_active())
            self.txtPrivateKey.set_sensitive(True)
            self.btnBrowse.set_sensitive(True)
            port = "22"
        else:
            self.get_widget("table2").hide_all()
            self.txtKeepAlive.set_text('0')
            self.txtKeepAlive.set_sensitive(False)
            self.chkKeepAlive.set_sensitive(False)
            self.chkX11.set_sensitive(False)
            self.chkAgent.set_sensitive(False)
            self.chkCompression.set_sensitive(False)
            self.txtCompressionLevel.set_sensitive(False)
            self.txtPrivateKey.set_sensitive(False)
            self.btnBrowse.set_sensitive(False)
            port = "23"
            if is_local:
                self.txtUser.set_text('')
                self.txtPassword.set_text('')
                self.txtPort.set_text('')
                self.txtHost.set_text('')
        self.txtPort.set_text(port)

    def on_chkKeepAlive_toggled(self, widget, *args):
        if widget.get_active():
            self.txtKeepAlive.set_text('120')
        else:
            self.txtKeepAlive.set_text('0')
        self.txtKeepAlive.set_sensitive(widget.get_active())

    def on_chkCompression_toggled(self, widget, *args):
        self.txtCompressionLevel.set_text('')
        self.txtCompressionLevel.set_sensitive(widget.get_active())

    def on_chkDynamic_toggled(self, widget, *args):
        self.txtRemoteHost.set_sensitive(not widget.get_active())
        self.txtRemotePort.set_sensitive(not widget.get_active())

    def on_btnAdd_clicked(self, widget, *args):
        local = self.txtLocalPort.get_text().strip()
        host = self.txtRemoteHost.get_text().strip()
        remote = self.txtRemotePort.get_text().strip()

        if self.chkDynamic.get_active():
            host = '*'
            remote = '*'

        # Validar datos del tunel
        if host == "":
            msgbox(_("Enter remote host"))
            return

        for x in self.treeModel:
            if x[0] == local:
                msgbox(_("Local port already assigned"))
                return

        tunel = self.treeModel.append(
            [local, host, remote,
             '%s:%s:%s' % (local, host, remote)])

    def on_btnDel_clicked(self, widget, *args):
        if self.treeTunel.get_selection().get_selected()[1] != None:
            self.treeModel.remove(
                self.treeTunel.get_selection().get_selected()[1])

    def on_chkCommands_toggled(self, widget, *args):
        self.txtCommands.set_sensitive(widget.get_active())

    def on_btnBColor_clicked(self, widget, *args):
        widget.selected_color = widget.get_color().to_string()

    def on_chkDefaultColors_toggled(self, widget, *args):
        self.btnFColor.set_sensitive(not widget.get_active())
        self.btnBColor.set_sensitive(not widget.get_active())

    def on_btnFColor_clicked(self, widget, *args):
        widget.selected_color = widget.get_color().to_string()

    def on_btnBrowse_clicked(self, widget, *args):
        filename = show_open_dialog(parent=self.main_widget,
                                    title=_("Open"),
                                    action=gtk.FILE_CHOOSER_ACTION_OPEN)
        if filename != None:
            self.txtPrivateKey.set_text(filename)