Пример #1
0
    def _apt_purge(self, package_name):
        purged = 0

        purge_command = [self.APT_GET_EXE, 'purge', '-y', package_name]

        try:
            result, err = self.utilcmds.run_command(purge_command)

            if err:
                raise Exception(err)

            found = re.search('\\d+ to remove', result)
            if found:
                amount_purged = found.group().split(' ')[0]
                purged = int(amount_purged)

            if purged > 0:
                logger.debug(
                    'Successfuly removed {0} packages.'.format(purged)
                )

                return 'true', ''
            else:
                logger.info('No packages were removed.')

                return 'false', 'No packages removed.'

        except Exception as e:
            logger.error('Problem while uninstalling package: ' + package_name)
            logger.exception(e)

            return 'false', str(e)
Пример #2
0
def load_queue(file_path):

    if not os.path.exists(file_path):
        return OperationQueue()

    loaded = []

    try:

        with open(file_path, 'r') as _file:
            loaded = cPickle.load(_file)

    except Exception as e:
        logger.error("Failed to load operations from: {0}".format(file_path))
        logger.exception(e)
        loaded = []

    logger.debug("Loaded operations: {0}".format(loaded))

    q = OperationQueue()

    for operaion in loaded:
        q.put(operaion)

    return q
Пример #3
0
    def _result_queue_loop(self):

        while True:
            self.result_queue_file_dump()

            queue_dump = self._result_queue.queue_dump()

            should_send = [result_op for result_op in queue_dump
                           if result_op.should_be_sent()]

            if should_send:

                logger.debug("Results to be sent: {0}".format(should_send))

                for result_op in should_send:
                    # TODO: what should be done if fails to remove?
                    self._result_queue.remove(result_op)
                    self.process_result_operation(result_op)

                self._result_queue.done()

            else:
                #logger.debug(
                #    "Results in queue: {0}".format(queue_dump)
                #)
                time.sleep(4)
Пример #4
0
    def _move_pkgs(self, install_data, app_plist_data):
        """ Move all pkgs in src to dest. """

        try:
            product_key = app_plist_data["productKey"]

            src = os.path.join(settings.UpdatesDirectory, install_data.id)
            dest = os.path.join("/Library/Updates", product_key)

            if not os.path.exists(dest):
                self._make_dir(dest)
                time.sleep(3)

            for _file in os.listdir(src):
                if _file.endswith(".pkg"):

                    su_pkg_path = os.path.join(dest, _file)
                    if os.path.exists(su_pkg_path):
                        os.remove(su_pkg_path)
                        logger.debug("Removed existing pkg from /Library/Updates: %s " % su_pkg_path)

                    src_pkg = os.path.join(src, _file)
                    shutil.move(src_pkg, dest)
                    logger.debug("Moved " + _file + " to: " + dest)

        except Exception as e:
            logger.error("Failed moving pkgs to /Library/Updates.")
            logger.exception(e)
            raise
Пример #5
0
    def _apt_install(self, package_name):
        logger.debug('Installing {0}'.format(package_name))

        install_command = [self.APT_GET_EXE, 'install', '-y', package_name]

        #TODO: figure out if restart needed
        restart = 'false'

        # TODO: parse out the error, if any
        try:
            result, err = self.utilcmds.run_command(install_command)

            if err:
                # Catch non-error related messages
                if 'reading changelogs' in err.lower():
                    pass
                else:
                    raise Exception(err)

        except Exception as e:
            logger.error('Faled to install {0}'.format(package_name))
            logger.exception(e)

            return 'false', str(e), restart

        logger.debug('Done installing {0}'.format(package_name))

        return 'true', '', restart
Пример #6
0
    def do_get_task_total(self, tqname=None, namespace='mobile', account=1, version=1, tags=None):
        '''
        Get total number of the specific task queue.

        Params:
            tqname: Name of task queue.
            namespace: Namespace of the task queue.
            account: Account ID.
            version: Version of the resource, default value 1.
            tags: Search conditions.

        Return:
            Total number of tasks on this taskqueue
            Exception message
        '''
        assert(tqname)
        resource = TQRES_TEMPLATE_STR % (version, account, namespace, tqname)
        resource = '/'.join([resource, 'stats'])
        if tags:
            data = json.dumps({'tags' : tags}, encoding="utf-8")
        else:
            data = tags
        logger.debug('Get total tasks number - resource:%s, data:%s' % (resource, data))
        result = self.m_wrest.do_access(resource, 'POST', data=data, headers=None)
        if result.code >= 200 and result.code < 300:
            logger.debug('Get Task Queue stats success!')
            try: #parse stat data
                tqs = json.loads(result.content)
                total_tasks = tqs.get('total_tasks')
                return total_tasks
            except Exception, e:
                logger.error('Get Task Queue stats Failed! %s' % repr(e))
Пример #7
0
    def do_bulk_add(self, tqname=None, namespace='mobile', account=1, version=1, taskslist=[]):
        '''
        Insert tasks into the specific task queue.

        Params:
            tqname: Name of task queue.
            namespace: Namespace of the task queue.
            account: Account ID.
            version: Version of the resource, default value 1.
            taskslist: task structures in list. 
                For example: [
                                {
                                    "payload_base64": "TXkgbGl0dGxlIHRhc2sK",
                                    "tags": ["state=red", "watchthis", "code=41"],
                                    "priority" : "high"
                                },
                                ...
                            ]

        Return:
            Instance object of class RestResult;
            Exception message
        '''
        #check data is available
        assert(tqname)
        assert(isinstance(taskslist, list))
        for task in taskslist:
            assert(isinstance(task, dict))
        resource = TQRES_TEMPLATE_STR % (version, account, namespace, tqname)
        resource = '/'.join([resource, 'addtasks'])
        data = json.dumps(taskslist, encoding="utf-8")
        logger.debug('Add %d tasks - resource:%s, data:%s' % (len(taskslist), resource, data))
        return self.m_wrest.do_access(resource, 'POST', data=data, headers=None)
Пример #8
0
    def do_lease(self, tqname=None, namespace='mobile', account=1, version=1, tasknum=0, tags=None, leasesec=10):
        '''
        Acquire a lease on the topmost {numTask} unowned tasks in the specified queue.

        Params:
            tqname: Name of task queue.
            namespace: Namespace of the task queue.
            account: Account ID.
            version: Version of the resource, default value 1.
            tasknum: Number of the tasks that expected.
            tags: Search conditions.

        Return:
            Instance object of class RestResult;
            Exception message
        '''
        assert(tqname)
        resource = TQRES_TEMPLATE_STR % (version, account, namespace, tqname)
        resource = '/'.join([resource, 'lease?numTasks=%s'])
        resource = resource % tasknum
        resource = '&'.join([resource, 'leaseSecs=%s'])
        resource = resource % leasesec
        if tags:
            data = json.dumps({'tags' : tags}, encoding="utf-8")
        else:
            data = tags
        logger.debug('Lease tasks - resource:%s, data:%s' % (resource, data))
        return self.m_wrest.do_access(resource, 'POST', data=data, headers=None)
Пример #9
0
    def do_add(self, tqname=None, namespace='mobile', account=1, version=1, payload=None, tags=None, priority='low', delay=0):
        '''
        Insert one task into the specific task queue.

        Params:
            tqname: Name of task queue.
            namespace: Namespace of the task queue.
            account: Account ID.
            version: Version of the resource, default value 1.
            payload: Base64 encode string.
            tags: Tags in the task.
            priority: Priority of the task.
            delay: If delay is set, the lease request will not see the added task in [delay] seconds.

        Return:
            Instance object of class RestResult;
            Exception message
        '''
        assert(tqname)
        resource = TQRES_TEMPLATE_STR % (version, account, namespace, tqname)
        resource = '/'.join([resource, 'addtask'])
        if delay > 0:
            resource += '?delay=%s' % delay
        data = json.dumps({'payload_base64' : payload, 'tags' : tags, 'priority' : priority}, encoding="utf-8")
        logger.debug('Add task - resource:%s, data:%s' % (resource, data))
        return self.m_wrest.do_access(resource, 'POST', data=data, headers=None)
Пример #10
0
    def do_update(self, tqname=None, namespace='mobile', account=1, version=1, taskname=0, payload=None, tags=None, priority='low', enqueuetime=''):
        '''
        Modify seciton value of one task.

        Params:
            tqname: Name of task queue.
            namespace: Namespace of the task queue.
            account: Account ID.
            version: Version of the resource, default value 1.
            taskname: Name of the specific task.
            payload: Base64 encode string.
            tags: Tags in the task.
            priority: Priority of the task.
            enqueuetime: Last modify time.

        Return:
            Instance object of class RestResult;
            Exception message
        '''
        assert(tqname)
        assert(payload and tags)
        resource = TQRES_TEMPLATE_STR % (version, account, namespace, tqname)
        temp = 'task-%s' % taskname
        resource = '/'.join([resource, temp])
        data = json.dumps({'payload_base64' : payload, 'tags' : tags, 'priority' : priority}, encoding="utf-8")
        header = self.m_wrest.generate_default_header()
        header.update({'If-Unmodified-Since' : enqueuetime})
        logger.debug('Update task - resource:%s, header:%s, data:%s' % (resource, header, data))
        return self.m_wrest.do_access(resource, 'PUT', data=data, headers=header)
Пример #11
0
def download(version=None):
	"""
    Download App Engine SDK
    """
	if not version:
		version = get_latest_version()
	if type(version) is str:
		version = tuple(version.split('.'))

	response = requests.get('https://storage.googleapis.com/appengine-sdks')
	response.raise_for_status()
	tree = ET.fromstring(response.text)

	path = None
	for key in tree.iter('{http://doc.s3.amazonaws.com/2006-03-01}Key'):
		match = re.match('^.*google_appengine_{0}.{1}.{2}\.zip$'.format(*version), key.text)
		if match:
			path = key.text
			break

	url = 'https://storage.googleapis.com/appengine-sdks/{path}'.format(**locals())

	logger.debug(' * Starting SDK download for version {0}.{1}.{2}'.format(*version))
	response = requests.get(url)
	response.raise_for_status()

	temp_zip = os.path.join(gettempdir(), 'google_appengine_{0}.{1}.{2}.zip'.format(*version))
	writefile(temp_zip, response.content, encode=None)
	return temp_zip
Пример #12
0
    def do_get_many(self, tqname=None, namespace='mobile', account=1, version=1, tasknum=0, tags=None):
        '''
        Get a bulk of tasks' information from the specific task queue.

        Params:
            tqname: Name of task queue.
            namespace: Namespace of the task queue.
            account: Account ID.
            version: Version of the resource, default value 1.
            tasknum: Number of the tasks that expected.
            tags: Search conditions.

        Return:
            Instance object of class RestResult;
            Exception message
        '''
        assert(tqname)
        resource = TQRES_TEMPLATE_STR % (version, account, namespace, tqname)
        resource = '/'.join([resource, 'search?numTasks=%s'])
        resource = resource % tasknum
        resource = '&'.join([resource, 'orderby=priority'])
        if tags:
            data = json.dumps({'tags' : tags}, encoding="utf-8")
        else:
            data = tags
        logger.debug('Get tasks - resource:%s, data:%s' % (resource, data))
        return self.m_wrest.do_access(resource, 'POST', data=data, headers=None)
Пример #13
0
    def do_insert(self, devinfo=None):
        '''
        Insert one new device information to RS

        Params:
            devinfo: Detail information of the device.

        Return:
            Instance object of class RestResult;
            Exception message
        '''
        if not self.m_udid:
            raise Exception('Require device UDID!')
        resource = DEVRES_TEMPLATE_STR % (self.m_ver, self.m_udid)
        if isinstance(devinfo, dict):
            temp = devinfo
        elif isinstance(devinfo, basestring):
            temp = json.loads(devinfo)
        else:
            raise Exception('Invalid input!')

        temp = {'attributes':temp}
        data = json.dumps(temp)

        wrest = HostedAccess()
        logger.debug('Insert new device info - resource:%s, data:%s' % (resource, data))
        retval = wrest.do_access(resource, 'PUT', data=data, headers=None)
        del wrest
        return retval
Пример #14
0
    def run_operation(self, operation):
        """Executes an operation given to it by the agent core.

        Returns:

            - Nothing

        """

        logger.debug("agent-id: {0}, agent-version: {1}"
                     .format(settings.AgentId, settings.AgentVersion))

        if not isinstance(operation, MonitOperation):
            operation = MonitOperation(operation.raw_operation)

        if operation.type == MonitOperationValue.MonitorData:
            monit_data = self.get_monit_data()

            operation.raw_result = json.dumps(monit_data)
            operation.urn_response = MonitUrn.get_monit_data_urn()
            operation.request_method = RequestMethod.POST

        else:
            logger.warning("Unknown operation %s. Ignoring." % operation.type)

        self._send_results(operation, retry=False)
Пример #15
0
    def do_update(self, udid=None, attributes=None):
        '''
        Update one device information to RS

        Params:
            attributes: Attributes of the devicd information.

        Return:
            Instance object of class RestResult;
            Exception message
        '''
        if udid:
            udid_str = str(udid)
        else:
            udid_str = self.m_udid
        if not udid_str:
            raise Exception('Require device UDID!')
        resource = DEVRES_TEMPLATE_STR % (self.m_ver, udid_str)
        if isinstance(attributes, list):
            attr = json.dumps(attributes)
        elif isinstance(attributes, basestring):
            attr = attributes
        else:
            raise Exception('Invalid input!')

        content = {
                        'method':'modify',
                        'params':attributes,
                    }
        data = json.dumps(content)
        wrest = HostedAccess()
        logger.debug('Update device info - resource:%s, data:%s' % (resource, data))
        retval = wrest.do_access(resource, 'POST', data=data, headers=None)
        del wrest
        return retval
Пример #16
0
    def do_get_many(self, idx, max, sortby, **conditions):
        '''
        Get devices' information from RS

        Params:
            idx: Page index number. (0 ~ )
            max: Page size.
            sortby: Sort condition string.
            conditions: The conditions for device selecting.

        Return:
            One dict with devices information.
            Exception message
        '''
        offset_val = idx * max + 1
        after_val = max - 1
        vlv_str = '{"sort":["%s"],"vlv":{"before":0,"after":%d,"offset":%d},' % (sortby, after_val, offset_val)
        resource = DEVSEARCH_TEMPLATE_STR % self.m_ver
        method = 'POST'
        post_data = ''.join([vlv_str, '"base":"ou=devices", "filter":"(&(objectClass=mobileDevice)'])
        if conditions:
            for k,v in conditions.iteritems():
                cond_str = '(%s=%s)' % (k, str(v))
                post_data = ''.join([post_data, cond_str])
        post_data = ''.join([post_data, ')"}'])
        logger.debug('Get %d devices info - resource:%s, data:%s' % (max, resource, repr(post_data)))
        wrest = HostedAccess()
        devinfo = wrest.do_access(resource, method, data=post_data, headers=None)
        del wrest
        return devinfo
Пример #17
0
    def do_bulk_op(self, content):
        '''
        Bulk add or/and update device information into RS

        Params:
            content:    data in a specific format, generated by format_bulk_data

        Return:
            instance object of class RestResult;
            exception message
        '''
        if isinstance(content, list):
            data = json.dumps(content)
        elif isinstance(attributes, basestring):
            data = content
        else:
            raise Exception('Invalid content!')

        resource = BULK_OP_URL % self.m_ver

        wrest = HostedAccess()
        logger.debug('bulk operation info - resource:%s, data:%s' % (resource, data))
        retval = wrest.do_access(resource, 'POST', data=data, headers=None)
        del wrest
        return retval
Пример #18
0
def resolve(address, uuid, adapter=None, bus=dbus.SystemBus()):
	try:
	    adapter.CreateDevice(address)
	    logger.debug("Device created: %s" % address)
	except:
	    logger.debug("Device all ready known: %s" % address)
	    
	path = adapter.FindDevice(address)
	
	device = dbus.Interface(bus.get_object("org.bluez", path),
							"org.bluez.Device")
	properties = device.GetProperties()
	
	if uuid in properties['UUIDs']:
	    services = device.DiscoverServices(uuid);
		
	    for key in services.keys():
		root=etree.XML(str(services[key]))
		res=CHANNEL_XPATH(root)
		if len(res)>0:
		    return int(res[0], 16)
	
	# lets check if the service is there now
	#adapter.RemoveDevice(path)
	
	raise Exception("UUID not found")
Пример #19
0
    def do_get(self, **conditions):
        '''
        Get one device information from RS

        Params:
            conditions: The conditions for device selecting.

        Return:
            One dict with device information.
            Exception message
        '''
        if self.m_udid:  # search by udid
            resource = DEVRES_TEMPLATE_STR % (self.m_ver, self.m_udid)
            method = 'GET'
            post_data = None
        else:  # without udid
            if conditions:
                resource = DEVSEARCH_TEMPLATE_STR % self.m_ver
                method = 'POST'
                post_data = '{"base":"ou=devices", "filter":"(&(objectClass=mobileDevice)'
                for cond in conditions:
                    temp_str = '(%s=%s)' % (str(cond), str(conditions[cond]))
                    post_data = post_data + temp_str
                post_data = ''.join([post_data,')"}'])
            else:
                raise Exception('Require device UDID or conditions for searching!')
        logger.debug('Get one device info - resource:%s' % (resource))
        wrest = HostedAccess()
        devinfo = wrest.do_access(resource, method, data=post_data, headers=None)
        del wrest
        return devinfo
Пример #20
0
    def get(self):
        """
        Attempts to get an operation from the queue if no operation is pending.

        Returns:
            The operation if it was successfully retrieved, None otherwise.

        """
        operation = None

        if (not self.op_in_progress) and (not self.paused):

            try:
                operation = self.queue.get_nowait()
                self.op_in_progress = True

                try:
                    logger.debug(
                        "Popping {0} from OpQueue.".format(operation.id)
                    )
                except Exception:
                    logger.debug("Popping {0} from OpQueue.".format(operation))

            except Queue.Empty as e:
#                logger.debug("Operations queue is empty.")
                operation = None

            except Exception as e:
                logger.error("Error accessing operation queue.")
                logger.error("Message: %s" % e)
                operation = None

        return operation
Пример #21
0
    def add_update_data(self, name, needs_restart=False):
        """
        Adds a new install item to the database.

        @param name: The vendor id this item refers to.
        @param needs_restart: True if restart required. False otherwise.
        @return: Nothing
        """

        if self.update_exist(name):
            logger.debug("Update %s already exist. Ignoring." % name)
            return

        _connection = self._connection
        with _connection:

            cursor = _connection.cursor()

            values = (name,
                      needs_restart)

            cursor.execute("INSERT INTO %s (%s) "
                           "VALUES (?, ?)" %
                           (self._table,
                            UpdateDataColumn.AllColumns), values)
Пример #22
0
    def move_task_to_errorqueue(self, error_msg):
        hostedTQ = HostedTaskQueue()
        if type(self.m_tags)!=list:
            logger.error('task tags is not a list:[%s]' % self.m_tags)
            raise Exception('tags type error in move_task_to_errorqueue')
        if TASK_STATUS_DICT['normal'] in self.m_tags:
            self.m_tags.remove(TASK_STATUS_DICT['normal'])
        if TASK_STATUS_DICT['retry'] in self.m_tags:
            self.m_tags.remove(TASK_STATUS_DICT['retry'])
        self.m_tags.append(TASK_STATUS_DICT['error'])

        tag_error = 'error_msg_base64=%s' % base64.encodestring(error_msg).replace('\n', '')
        self.m_tags.append(tag_error)
        try:
            hostedTQ.do_add(tqname=self.m_error_qname, namespace='mobile', account=int(self.m_account), version=1, payload=self.m_payload_base64, tags=self.m_tags, priority = self.m_priority)
        except MDMiHttpError, e:
            logger.debug('add error task failed:%s, will try to create error queue' % repr(e))
            error_qname_list = [
            {'name' : 'enrollmentError', 'settings' : {"description" : "Enrollment Decision Error task queue", "max_leases" : "30", "max_age" : "0"}}
            ]
            tqo = TaskQueueUtils(error_qname_list)
            tqo.init_task_queue()
            del tqo

            hostedTQ.do_add(tqname=self.m_error_qname, namespace='mobile', account=int(self.m_account), version=1, payload=self.m_payload_base64, tags=self.m_tags, priority = self.m_priority)
Пример #23
0
def list_sdk_versions(args):
    logger.debug(' * Retreiving list of App Engine SDK versions')
    versions = sdk.get_versions()
    for version in sdk.get_versions():
        print '{0}.{1}.{2}'.format(*version)

    sys.exit(0)
Пример #24
0
    def install_update(self, install_data, update_dir=None):
        logger.debug('Received install_update call.')

        old_install_list = self.get_installed_applications()

        success = 'false'
        error = ''
        restart = 'false'
        app_encoding = CreateApplication.null_application().to_dict()
        apps_to_delete = []
        apps_to_add = []

        success, error, restart = self._yum_update(install_data.name)

        if success == 'true':

            new_install_list = self.get_installed_applications()

            app = self._get_installed_app(install_data.name, new_install_list)
            app_encoding = app.to_dict()

            apps_to_add, apps_to_delete = self._get_apps_to_add_and_delete(
                old_install_list, new_install_list
            )

        return InstallResult(
            success,
            error,
            restart,
            app_encoding,
            apps_to_delete,
            apps_to_add
        )
Пример #25
0
    def setup_table(self, num):
        table = 'test_%s' % num
        self.cursor.execute('SHOW TABLES LIKE %s', [table])
        if self.cursor.fetchone():
            # Table already exists.
            return table

        logger.debug('Creating table "%s"' % table)

        _dir = dirname(__file__) or '.'
        with open(_dir + '/mysql_mid_table.sql', 'rb') as f:
            sql = f.read() % num
        self.cursor.execute(sql.strip(' ;\r\n'))

        delta = timedelta(seconds=self.CREATED_INTERVAL)
        start_dt = datetime.now() - delta * num
        for chunk in xrange(int(ceil(float(num) / self.CHUNK_SIZE))):
            values = list(chain(*[[
                MID_RECORD['name'] % (i + chunk * self.CHUNK_SIZE),
                MID_RECORD['data'],
                start_dt + delta * i,
            ] for i in xrange(self.CHUNK_SIZE)]))
            sql = (
                'INSERT INTO `%s` (`name`, `data`, `created`) VALUES ' % table
            ) + ('(%s, %s, %s), ' * self.CHUNK_SIZE)[:-2]
            self.cursor.execute('BEGIN')
            self.cursor.execute(sql, values)
            self.cursor.execute('COMMIT')
#             logger.debug(
#                 'Inserted chunk #%s with %s middle sized records',
#                 chunk + 1, self.CHUNK_SIZE
#             )
        logger.debug('Created with %s records' % num)
        return table
Пример #26
0
	def _get(self, method, args):
		"""
			private helper to perform an api call
		"""
		url = self.API_BASE % (self.api_key, method, "/".join(args))
		logger.debug("fetching url '%s'" % url)
		return self._get_url(url)
Пример #27
0
def save_package(package):

    with _connection:

        cursor = _connection.cursor()

        values = (package.name,
                  package.version_release,
                  package.description,
                  package.support_url,
                  package.severity,
                  package.toppatch_id,
                  package.vendor_id,
                  package.date_installed,
                  package.date_published,
                  package.is_update,
                  package.package_url)

        logger.debug("Adding TPID# %s" % package.toppatch_id)
        cursor.execute("INSERT INTO %s (name, version_release, description,"
                       "support_url, severity, toppatch_id, vendor_id,"
                       "date_installed, date_published, is_update, package_url)"
                       " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)" %
                       _packages_table,
                       values)
Пример #28
0
    def run(self):
        # read keys from files at first
        if self.m_enable_init:
            logger.info('Creator %d running, doing init!' % os.getpid())
            keys_list = self._init_keys_list()
            for item in keys_list:
                if not self.m_keylist_queue.full():
                    self._put(item)

        else:
            # waiting for init finished
            logger.debug('Creator %d running, waiting init done!' % os.getpid())
            while not self.m_ready_evt.is_set():
                if self._idle(1):
                    return
        logger.info('Creator %d init done!' % os.getpid())

        self._generate_keys_handle()

        # clean multiprocess queue
        self.m_keylist_queue.close()
        while not self.m_keylist_queue.empty():
            self.m_keylist_queue.get(1)

        logger.info('key pool creator(%d) exits' % os.getpid())
        return
        pass
Пример #29
0
 def put_data(self, data, block=True, timeout=0):
     task_data = {}
     
     data = json.loads(data)
     if data['sync_type'] == 'metanate':
         key = ':'.join(['metanate', str(data['account']), data['trans_type'], data['sync_source']])
         task_data['key'] = key
         task_data['value'] = data
     elif data['sync_type'] == 'hybrid':
         key = ':'.join([data['sync_type'], str(data['account'])])
         task_data['key'] = key
         task_data['value'] = data
         
     while not self.__stop_event.is_set():
         try:
             self.__queue.put(task_data, block, timeout)
             logger.debug('put task data (%s, %s) successfully' % (key, data))
             break
         except Full:
             logger.debug('task cache size reachs max size')
             self.full_event.clear()
             self.full_event.wait()
         except Exception, e:
             logger.error('put task data into cache error %s' % repr(e))
             break
Пример #30
0
def signal_handler(signum, frame):
    logger.debug('Catched interrupt signal: %d in main process', signum)
    if signum == signal.SIGHUP:
        reload_config()
        g_service_status.reload_status()
    else:
        stop_children()
Пример #31
0
 def restartPollers(self):
     log.debug("%s: restarting pollers", self.monitorId)
     for poller in self.pollerSet:
         if poller.init is not None:          
             poller.init(poller)
Пример #32
0
 def _signal_handler(self, signum, frame):
     logger.debug('Catched interrupt signal in socket process: %d', signum)
     self.is_teminate = True
Пример #33
0
 def signal_handler(signal, frame):
     logger.debug('Terminating PhantomJS before exit')
     self.driver.quit()
     sys.exit(0)
Пример #34
0
    def get_article_obj_from_url(url):
        logger.debug('Fetching content from url: [{}]'.format(url))
        article_page_content = get_content(url)
        logger.debug('Fetched article page')

        article_div = bs(
            article_page_content,
            'html.parser').body.find('main').div.div.div.div.div.div.article
        logger.debug('Parsed article tag')

        # For debugging
        # write_html('tmp.html', str(article_div))

        title = article_div.find('h1').string.strip()
        logger.debug('Fetched title: [{}]'.format(title))

        author = article_div.find('div', {
            'class': 'post-meta__author-name'
        }).string.strip()
        logger.debug('Fetched author: [{}]'.format(author))

        date_tag = article_div.find('div', {
            'class': 'post-meta__publish-date'
        }).time
        date = date_tag['datetime']
        diff = date_tag.string.strip()
        logger.debug('Fetched date: [{}] and diff: [{}]'.format(date, diff))

        stats_div = article_div.find(
            'div',
            {'class': 'post-actions post__block post__block_post-actions'})
        views = -1
        shares = -1
        for div in stats_div.find_all(
                'div',
            {'class': 'post-actions__item post-actions__item_stat'}):
            name = div.find('span', {
                'class': 'post-actions__item-title'
            }).string.strip().lower()
            number = int(
                div.find('span', {
                    'class': 'post-actions__item-count'
                }).string.strip())
            if name.find('views') > -1:
                views = number
            else:
                shares = number
        logger.debug('Fetched views: [{}] and shares [{}]'.format(
            views, shares))

        tags_div = article_div.find('ul', {
            'class': 'tags-list__list'
        }).find_all('li')
        tags = [tag.a.string.strip() for tag in tags_div]
        logger.debug('Fetched tags: [{}]'.format(tags))

        return Article(url, title, author, views, shares, date, diff, tags)
Пример #35
0
 def from_html(cls, html_filepath):
     logger.debug(
         'Collecting data from HTML file: [{}]'.format(html_filepath))
     return cls(ArticleParser.get_urls_from_html(html_filepath))
Пример #36
0
 def from_csv(cls, csv_filepath):
     logger.debug(
         'Collecting data from CSV file: [{}]'.format(csv_filepath))
     return cls(ArticleParser.get_urls_from_csv(csv_filepath))
Пример #37
0
 def validate_urls(self):
     logger.debug('Validating URLs')
     self.urls = list(filter(is_url_valid, self.urls))
     logger.debug('URL validation complete')
Пример #38
0
 def add_article(self, article, add_to_file=True):
     self.articles.append(article)
     if add_to_file:
         self.writer.writerow(article.__dict__)
         self.csv_file.flush()
     logger.debug('Article added')
Пример #39
0
 def run(self, output_fn, **kwargs):
     logger.debug(
         "---------------------------------------------------------------------------------"
     )
     logger.debug(
         "------------------------Welcome to Monitor Daemon.-------------------------------"
     )
     logger.debug(
         "------Copyright (2019, ) Institute of Software, Chinese Academy of Sciences------"
     )
     logger.debug(
         "---------author: [email protected],[email protected]"
     )
     logger.debug(
         "--------------------------------wuheng@otcaix.iscas.ac.cn------------------------"
     )
     logger.debug(
         "---------------------------------------------------------------------------------"
     )
     start_http_server(19998)
     #         registry = CollectorRegistry(auto_describe=False)
     config.load_kube_config(config_file=TOKEN)
     zone = get_field_in_kubernetes_node(HOSTNAME,
                                         ['metadata', 'labels', 'zone'])
     while True:
         #             init(registry)
         collect_vm_metrics(zone)
         collect_storage_metrics(zone)
         time.sleep(10)
Пример #40
0
def main(args):
    args.batch_size = None
    logger.debug('CONFIGURATION: {}'.format(args))
    """ Set up the graph, the agents, and run the agents in parallel. """
    if args.env == 'GYM':
        from environments import atari_environment
        num_actions, action_space, _ = atari_environment.get_actions(args.game)
        input_shape = atari_environment.get_input_shape(args.game)
    else:
        num_actions = get_num_actions(args.rom_path, args.game)

    args.action_space = action_space
    args.summ_base_dir = '/tmp/summary_logs/{}/{}'.format(
        args.game, time.strftime('%m.%d/%H.%M'))
    logger.info('logging summaries to {}'.format(args.summ_base_dir))

    Learner, Network = ALGORITHMS[args.alg_type]
    network = Network({
        'name': 'shared_vars_network',
        'input_shape': input_shape,
        'num_act': num_actions,
        'args': args
    })
    args.network = Network

    #initialize shared variables
    args.learning_vars = SharedVars(network.params)
    args.opt_state = SharedVars(
        network.params, opt_type=args.opt_type,
        lr=args.initial_lr) if args.opt_mode == 'shared' else None
    args.batch_opt_state = SharedVars(
        network.params, opt_type=args.opt_type,
        lr=args.initial_lr) if args.opt_mode == 'shared' else None

    #TODO: need to refactor so TRPO+GAE doesn't need special treatment
    if args.alg_type == 'trpo':
        baseline_network = PolicyValueNetwork(
            {
                'name': 'shared_value_network',
                'input_shape': input_shape,
                'num_act': num_actions,
                'args': args
            },
            use_policy_head=False)
        args.baseline_vars = SharedVars(baseline_network.params)

    if args.alg_type in ['q', 'sarsa', 'dueling', 'dqn-cts']:
        args.target_vars = SharedVars(network.params)
        args.target_update_flags = SharedFlags(args.num_actor_learners)

    args.barrier = Barrier(args.num_actor_learners)
    args.global_step = SharedCounter(0)
    args.num_actions = num_actions

    cuda_visible_devices = os.getenv('CUDA_VISIBLE_DEVICES')
    num_gpus = 0
    if cuda_visible_devices:
        num_gpus = len(cuda_visible_devices.split())

    #spin up processes and block
    if (args.visualize == 2): args.visualize = 0
    actor_learners = []
    task_queue = Queue()
    experience_queue = Queue()
    for i in xrange(args.num_actor_learners):
        if (args.visualize == 2) and (i == args.num_actor_learners - 1):
            args.args.visualize = 1

        args.actor_id = i
        args.device = '/gpu:{}'.format(i % num_gpus) if num_gpus else '/cpu:0'

        rng = np.random.RandomState(int(time.time()))
        args.random_seed = rng.randint(1000)

        #only used by TRPO
        args.task_queue = task_queue
        args.experience_queue = experience_queue

        args.input_shape = input_shape
        actor_learners.append(Learner(args))
        actor_learners[-1].start()

    try:
        for t in actor_learners:
            t.join()
    except KeyboardInterrupt:
        #Terminate with extreme prejudice
        for t in actor_learners:
            t.terminate()

    logger.info('All training threads finished!')
Пример #41
0
 def gain_honor(self, value):
     assert value >= 0
     self.honor += value
     logger.debug("Gain honor[value=%d]" % value)
Пример #42
0
 def delete_profile(self, *args):
     logger.debug(f'[UI] "{self.name}" Edit-Dialogue: Delete-Profile-Clicked')
     u.Profile(username=self.username).delete()
     self.dialogue.dismiss(force=True)
     self.screen.refresh_visible_profiles()
metadata_products = pd.read_csv(os.path.join(DIR, 'products.csv'))


with open(os.path.join(DIR, 'model.pkl'), 'rb') as f:
    model = joblib.load(f)

time_bechmark = []
scores = []
if __name__ == '__main__':
    for i in range(len(sample_queries)):
        query = json.loads(sample_queries.at[i, 'query'])
        actual = json.loads(sample_queries.at[i, 'next_trans'])['product_ids']
        if len(query['transaction_history']) == 0:
            continue

        start_time = time.time()
        products_hist_counter, histdata_products = extract_data_from_json_dict(query)
        recs = model.recommend(products_hist_counter, metadata_products, histdata_products)
        # recs = model.recommend(products_hist_counter)
        finish_time = time.time()

        score = normalized_average_precision(actual, recs)
        scores.append(score)
        # logger.debug(f'Query: {i}, processed in {(finish_time - start_time):.5f} seconds')
        logger.debug(f'Query {i}, NAP@30: {score}, Num transactions: {len(query["transaction_history"])}')
        time_bechmark.append(finish_time - start_time)

    logger.debug(f'Ran {len(sample_queries)} queries, Average elapsed time: {np.mean(time_bechmark)} ± {np.std(time_bechmark)}')
    logger.debug(f'MNAP@30: {np.mean(scores)}')
Пример #44
0
    def dumps(self,
              user_info,
              pac_url,
              exceptlst,
              pkey,
              vpn_bypass_except,
              sn=None):
        """
        Dumps vpn profile to plist string.
        Parameter Notes:
            user_info(string): must contains Username and DeviceID keyword,
                        like "Username:[email protected];DeviceID:someone's device"
            pack_url(string): a valid http or https url
        """
        p12, cn, expire_date = issuer_certificate(self.issuer_ca,
                                                  self.issuer_key, user_info,
                                                  config['passout'], pkey)
        if not p12 or not cn:
            return None, None

        data = dumps_p12_to_string(p12)
        if not data:
            return None, None

        self.activesync_ca_data = base64.encodestring(data)
        self.add_node(self.user_ca_payload, 'PayloadCertificateFileName', cn,
                      'string')
        self.add_node(self.user_ca_payload, 'PayloadDisplayName', cn, 'string')
        self.add_node(self.user_ca_payload, 'PayloadContent',
                      self.activesync_ca_data, 'data')

        node_proxy_val = self.get_next_node(self.node_vpn, 'Proxies')
        node_proxy_url = self.get_next_node(node_proxy_val,
                                            'ProxyAutoConfigURLString')
        node_proxy_url.text = self.update_pacfile_type(pac_url)

        if len(exceptlst) > 0:
            domain_match_node = self.get_next_node(
                self.node_ipsec_v, 'OnDemandMatchDomainsNever')
            d_node_val = self.get_domainaction_node()

            for vpnexcept in exceptlst:
                try:
                    assert vpnexcept.has_key(
                        'attributes'
                    ), 'vpn exception dict do not has key: attributes'
                    node_except_string = ElementTree.SubElement(
                        domain_match_node, 'string')
                    node_except_string1 = ElementTree.SubElement(
                        d_node_val, 'string')

                    if vpnexcept['attributes'].has_key('domainName'):
                        node_except_string.text = vpnexcept["attributes"][
                            "domainName"]
                        node_except_string1.text = vpnexcept["attributes"][
                            "domainName"]
                    elif vpnexcept['attributes'].has_key('ipaddress'):
                        node_except_string.text = vpnexcept["attributes"][
                            "ipaddress"]
                        node_except_string1.text = vpnexcept["attributes"][
                            "ipaddress"]
                    elif vpnexcept['attributes'].has_key('subnet'):
                        node_except_string.text = vpnexcept["attributes"][
                            "subnet"]
                        node_except_string1.text = vpnexcept["attributes"][
                            "subnet"]
                    else:
                        logger.debug(
                            'no domainName, ipaddress and subnet in vpn exception dict: %s'
                            % vpnexcept)
                except Exception, ve:
                    logger.error(
                        "add vpn exception value to vpn profile error %s" %
                        repr(ve))
    def train(self):
        """ Main actor learner loop for n-step Q learning. """
        logger.debug("Actor {} resuming at Step {}, {}".format(
            self.actor_id, self.global_step.value(), time.ctime()))

        s = self.emulator.get_initial_state()

        s_batch = list()
        a_batch = list()
        y_batch = list()
        bonuses = deque(maxlen=1000)
        episode_over = False

        t0 = time.time()
        global_steps_at_last_record = self.global_step.value()
        while (self.global_step.value() < self.max_global_steps):
            # # Sync local learning net with shared mem
            # self.sync_net_with_shared_memory(self.local_network, self.learning_vars)
            # self.save_vars()
            rewards = list()
            states = list()
            actions = list()
            max_q_values = list()
            local_step_start = self.local_step
            total_episode_reward = 0
            total_augmented_reward = 0
            episode_ave_max_q = 0
            ep_t = 0

            while not episode_over:
                # Sync local learning net with shared mem
                self.sync_net_with_shared_memory(self.local_network,
                                                 self.learning_vars)
                self.save_vars()

                # Choose next action and execute it
                a, q_values = self.choose_next_action(s)

                new_s, reward, episode_over = self.emulator.next(a)
                total_episode_reward += reward
                max_q = np.max(q_values)

                current_frame = new_s[..., -1]
                bonus = self.density_model.update(current_frame)
                bonuses.append(bonus)

                # Rescale or clip immediate reward
                reward = self.rescale_reward(
                    self.rescale_reward(reward) + bonus)
                total_augmented_reward += reward
                ep_t += 1

                rewards.append(reward)
                states.append(s)
                actions.append(a)
                max_q_values.append(max_q)

                s = new_s
                self.local_step += 1
                episode_ave_max_q += max_q

                global_step, _ = self.global_step.increment()

                if global_step % self.q_target_update_steps == 0:
                    self.update_target()
                if global_step % self.density_model_update_steps == 0:
                    self.write_density_model()

                # Sync local tensorflow target network params with shared target network params
                if self.target_update_flags.updated[self.actor_id] == 1:
                    self.sync_net_with_shared_memory(self.target_network,
                                                     self.target_vars)
                    self.target_update_flags.updated[self.actor_id] = 0
                if self.density_model_update_flags.updated[self.actor_id] == 1:
                    self.read_density_model()
                    self.density_model_update_flags.updated[self.actor_id] = 0

                if self.local_step % self.q_update_interval == 0:
                    self.batch_update()

                if self.is_master() and (self.local_step % 500 == 0):
                    bonus_array = np.array(bonuses)
                    steps = global_step - global_steps_at_last_record
                    global_steps_at_last_record = global_step

                    logger.debug(
                        'Mean Bonus={:.4f} / Max Bonus={:.4f} / STEPS/s={}'.
                        format(bonus_array.mean(), bonus_array.max(),
                               steps / float(time.time() - t0)))
                    t0 = time.time()

            else:
                #compute monte carlo return
                mc_returns = np.zeros((len(rewards), ), dtype=np.float32)
                running_total = 0.0
                for i, r in enumerate(reversed(rewards)):
                    running_total = r + self.gamma * running_total
                    mc_returns[len(rewards) - i - 1] = running_total

                mixed_returns = self.cts_eta * np.asarray(rewards) + (
                    1 - self.cts_eta) * mc_returns

                #update replay memory
                states.append(new_s)
                episode_length = len(rewards)
                for i in range(episode_length):
                    self.replay_memory.append(states[i], actions[i],
                                              mixed_returns[i],
                                              i + 1 == episode_length)

            s, total_episode_reward, _, ep_t, episode_ave_max_q, episode_over = \
                self.prepare_state(s, total_episode_reward, self.local_step, ep_t, episode_ave_max_q, episode_over, bonuses, total_augmented_reward)
Пример #46
0
    def _call(self, txn, commands):
        for sql in commands:
            logger.debug("SQL execute[sql=%s]" % sql)
            txn.execute(sql)

        return txn.fetchall()
Пример #47
0
 def update_food_capacity(self, capacity):
     """更新粮食储量
     """
     logger.debug("update food capacity[capacity=%f]" % capacity)
     assert capacity >= 0
     self.food_capacity = capacity
Пример #48
0
    def train(self):
        """ Main actor learner loop for advantage actor critic learning. """
        logger.debug("Actor {} resuming at Step {}".format(
            self.actor_id, self.global_step.value()))

        s = self.emulator.get_initial_state()
        steps_at_last_reward = self.local_step
        total_episode_reward = 0.0
        mean_entropy = 0.0
        q_update_counter = 0
        episode_start_step = 0

        while (self.global_step.value() < self.max_global_steps):
            # Sync local learning net with shared mem
            self.sync_net_with_shared_memory(self.local_network,
                                             self.learning_vars)
            self.save_vars()
            params = self.session.run(self.local_network.flat_vars)
            self.assign_vars(self.batch_network, params)

            local_step_start = self.local_step
            reset_game = False
            episode_over = False

            states = list()
            rewards = list()
            actions = list()
            values = list()
            s_batch = list()
            a_batch = list()
            y_batch = list()
            adv_batch = list()

            while not (episode_over or (self.local_step - local_step_start
                                        == self.max_local_steps)):

                # Choose next action and execute it
                a, readout_v_t, readout_pi_t = self.choose_next_action(s)

                if self.is_master() and (self.local_step % 100 == 0):
                    logger.debug("pi={}, V={}".format(readout_pi_t,
                                                      readout_v_t))

                new_s, reward, episode_over = self.emulator.next(a)

                if reward != 0.0:
                    steps_at_last_reward = self.local_step

                total_episode_reward += reward
                # Rescale or clip immediate reward
                reward = self.rescale_reward(reward)
                self.replay_memory.append(s, a, reward, episode_over)

                rewards.append(reward)
                states.append(s)
                actions.append(a)
                values.append(readout_v_t)

                s = new_s
                self.local_step += 1
                self.global_step.increment()

            # Calculate the value offered by critic in the new state.
            if episode_over:
                R = 0
            else:
                R = self.session.run(
                    self.local_network.output_layer_v,
                    feed_dict={self.local_network.input_ph: [new_s]})[0][0]

            sel_actions = []
            for i in reversed(xrange(len(states))):
                R = rewards[i] + self.gamma * R

                y_batch.append(R)
                a_batch.append(actions[i])
                s_batch.append(states[i])
                adv_batch.append(R - values[i])

                sel_actions.append(np.argmax(actions[i]))

            # Compute gradients on the local policy/V network and apply them to shared memory
            feed_dict = {
                self.local_network.input_ph: s_batch,
                self.local_network.critic_target_ph: y_batch,
                self.local_network.selected_action_ph: a_batch,
                self.local_network.adv_actor_ph: adv_batch,
            }
            grads, entropy = self.session.run(
                [self.local_network.get_gradients, self.local_network.entropy],
                feed_dict=feed_dict)

            self.apply_gradients_to_shared_memory_vars(grads)

            # q_grads = self.batch_q_update()
            # if q_grads is not None:
            #     grads = [p + q for p, q in zip(policy_grads, q_grads)]
            # else:
            #     grads = policy_grads
            q_update_counter += 1
            if q_update_counter % self.q_update_interval == 0:
                q_grads = self.batch_q_update()
                if q_grads is not None:
                    # grads = [p + q for p, q in zip(grads, q_grads)]
                    self.apply_gradients_to_shared_memory_vars(q_grads)

            delta_old = local_step_start - episode_start_step
            delta_new = self.local_step - local_step_start
            mean_entropy = (mean_entropy * delta_old +
                            entropy * delta_new) / (delta_old + delta_new)

            s, mean_entropy, mean_value, episode_start_step, total_episode_reward, steps_at_last_reward = self.prepare_state(
                s, mean_entropy,
                np.array(values).mean(), episode_start_step,
                total_episode_reward, steps_at_last_reward, sel_actions,
                episode_over)
Пример #49
0
 def update_food_output(self, output):
     """更新粮食产量
     """
     logger.debug("update food output[output=%f]" % output)
     assert output >= 0
     self.food_output = output
    def _train(self):
        """ Main actor learner loop for advantage actor critic learning. """
        logger.debug("Actor {} resuming at Step {}".format(
            self.actor_id, self.global_step.value()))

        bonuses = deque(maxlen=100)
        while (self.global_step.value() < self.max_global_steps):
            # Sync local learning net with shared mem
            s = self.emulator.get_initial_state()
            self.reset_hidden_state()
            self.local_episode += 1
            episode_over = False
            total_episode_reward = 0.0
            episode_start_step = self.local_step

            while not episode_over:
                self.sync_net_with_shared_memory(self.local_network,
                                                 self.learning_vars)
                self.save_vars()

                rewards = list()
                states = list()
                actions = list()
                values = list()
                local_step_start = self.local_step
                self.set_local_lstm_state()

                while self.local_step - local_step_start < self.max_local_steps and not episode_over:
                    # Choose next action and execute it
                    a, readout_v_t, readout_pi_t = self.choose_next_action(s)
                    new_s, reward, episode_over = self.emulator.next(a)
                    total_episode_reward += reward

                    # Update density model
                    current_frame = new_s[..., -1]
                    bonus = self.density_model.update(current_frame)
                    bonuses.append(bonus)

                    if self.is_master() and (self.local_step % 400 == 0):
                        bonus_array = np.array(bonuses)
                        logger.debug(
                            'π_a={:.4f} / V={:.4f} / Mean Bonus={:.4f} / Max Bonus={:.4f}'
                            .format(readout_pi_t[a.argmax()], readout_v_t,
                                    bonus_array.mean(), bonus_array.max()))

                    # Rescale or clip immediate reward
                    reward = self.rescale_reward(
                        self.rescale_reward(reward) + bonus)
                    rewards.append(reward)
                    states.append(s)
                    actions.append(a)
                    values.append(readout_v_t)

                    s = new_s
                    self.local_step += 1

                    global_step, _ = self.global_step.increment()
                    if global_step % self.density_model_update_steps == 0:
                        self.write_density_model()
                    if self.density_model_update_flags.updated[
                            self.actor_id] == 1:
                        self.read_density_model()
                        self.density_model_update_flags.updated[
                            self.actor_id] = 0

                next_val = self.bootstrap_value(new_s, episode_over)
                advantages = self.compute_gae(rewards, values, next_val)
                targets = self.compute_targets(rewards, next_val)
                # Compute gradients on the local policy/V network and apply them to shared memory
                entropy = self.apply_update(states, actions, targets,
                                            advantages)

            elapsed_time = time.time() - self.start_time
            steps_per_sec = self.global_step.value() / elapsed_time
            perf = "{:.0f}".format(steps_per_sec)
            logger.info(
                "T{} / EPISODE {} / STEP {}k / REWARD {} / {} STEPS/s".format(
                    self.actor_id, self.local_episode,
                    self.global_step.value() / 1000, total_episode_reward,
                    perf))

            self.log_summary(total_episode_reward,
                             np.array(values).mean(), entropy)
Пример #51
0
def signal_handler(signum, frame):
    global is_signal_up
    logger.debug('Catched a stop signal!')
    is_signal_up = True
    pass
Пример #52
0
 def update_money_capacity(self, capacity):
     """更新金钱储量
     """
     logger.debug("update money capacity[capacity=%f]" % capacity)
     assert capacity >= 0
     self.money_capacity = capacity
Пример #53
0
    def choose_rival(self, self_battle_score, rivals_user_id, rivals_win_score,
                     rivals_user_id_origin, rivals_battle_score):
        """选择对手
        Args:
            rivals_win_score: 三个对手分别打赢获得的积分
        """
        if len(rivals_win_score) != MAX_RIVALS_NUM:
            return False

        if self.continuous_win_num <= CONTINUS_WIN_NUM:
            #从第一个对手到第二个对手依次比较我方战力和敌方战力,如果我方战力超过敌方战力15%(可调),则直接匹配
            for i in range(len(rivals_user_id) - 1):
                if self.is_arena_rival_pve(rivals_user_id[i]):
                    logger.debug("rival is pve[rival_id=%d]" %
                                 rivals_user_id[i])
                    continue

                diff_ratio = 1.0 * (self_battle_score - rivals_battle_score[i]
                                    ) / rivals_battle_score[i]
                logger.debug(
                    "choose_rival[self_battle_score=%d][diff_ratio=%f][rival_battle_score=%d]]"
                    % (self_battle_score, diff_ratio, rivals_battle_score[i]))
                if diff_ratio > DIFF_BATTLE_SCORE_RATIO:
                    #选定此对手
                    for j in range(len(rivals_user_id_origin)):
                        if rivals_user_id_origin[j] == rivals_user_id[i]:
                            self.choose_rival_index = j
                            return True

        win_max = max(data_loader.ArenaChoosePlayer_dict.keys())
        win_min = min(data_loader.ArenaChoosePlayer_dict.keys())
        if self.continuous_win_num >= 0:
            num = min(self.continuous_win_num, win_max)
        else:
            num = max(self.continuous_win_num, win_min)

        ratio = []
        ratio.append(
            float(data_loader.ArenaChoosePlayer_dict[num].firstProbability) /
            100)
        ratio.append(
            float(data_loader.ArenaChoosePlayer_dict[num].secondProbability) /
            100 + ratio[0])
        ratio.append(
            float(data_loader.ArenaChoosePlayer_dict[num].threeProbability) /
            100 + ratio[1])

        #roll
        choose_index = 0
        random.seed()
        c = random.random()
        for index, r in enumerate(ratio):
            if c < r:
                choose_index = index
                break

        for i in range(len(rivals_user_id_origin)):
            if rivals_user_id_origin[i] == rivals_user_id[choose_index]:
                self.choose_rival_index = i
                break
        logger.debug("roll rival[roll=%f][ratio1=%f][ratio2=%f][ratio3=%f]" %
                     (c, ratio[0], ratio[1], ratio[2]))

        return True
Пример #54
0
 def update_money_output(self, output):
     """更新金钱产量
     """
     logger.debug("update money output[output=%f]" % output)
     assert output >= 0
     self.money_output = output
Пример #55
0

if __name__ == "__main__":

    import argparse
    parser = argparse.ArgumentParser()
    parser.add_argument('--distmap', default=False, action="store_true")
    parser.add_argument('--final', default=False, action="store_true")
    parser.add_argument('--depth', default=False, action="store_true")
    parser.add_argument('--iou', default=False, action="store_true")
    parser.add_argument('--debug', default=True, action="store_true")
    parser.add_argument(
        '--input',
        default="results/cityscapes/test/cityscapes-munster_000061_000019")
    args = parser.parse_args()
    logger.debug(args)

    lwmap_range = (0, 10)

    if args.debug: enable_debug()

    if args.distmap:
        create_folder("results/distmap/")
        gt_path = f'{args.input}-gt_affordances.png'
        vis_path = f'{args.input}-orig-rgb_affordances.png'
        depth_path = 'datasets/freiburg-forest/freiburg_forest_multispectral_annotated/freiburg_forest_annotated/test/depth_gray/b1-09517_Clipped_redict_depth_gray.png'
        pred_path = gt_path
        image_orig = imread(gt_path)
        image_orig = cv2.resize(image_orig, dsize=(480, 240))

        if args.depth:
Пример #56
0
def check_hero(data,
               message,
               with_level=False,
               with_star=False,
               with_soldier=False,
               with_equipment_type=0,
               with_skill_index=-1,
               with_evolution=False,
               with_equipment_stone=False,
               with_herostar=False,
               with_awaken=False):
    """
    验证英雄信息
    Args:
        data[UserData]
        message[protobuf]
        with_xxx[bool] 是否需要验证某些字段,包括:等级、星级、兵种、装备、技能
    """
    hero_id = HeroInfo.generate_id(data.id, message.basic_id)
    hero = data.hero_list.get(hero_id, True)
    assert hero is not None

    if with_level:
        logger.debug("check hero[level=%d][exp=%d]" % (hero.level, hero.exp))
        assert hero.level == message.level
        #客户端bug导致此处经常失败,先放过
        #assert hero.exp == message.exp
        if hero.exp != message.exp:
            logger.warning(
                "check hero error[basic_id=%d][exp=%d][req_exp=%d]" %
                (message.basic_id, hero.exp, message.exp))
    if with_star:
        assert hero.star == message.star_level
    if with_soldier:
        assert hero.soldier_basic_id == message.soldier_basic_id
        assert hero.soldier_level == message.soldier_level
    if with_equipment_type != 0 and with_equipment_stone != True:
        if with_equipment_type == HeroInfo.EQUIPMENT_TYPE_WEAPON:
            assert hero.get_equipment(
                with_equipment_type) == message.equipment_weapon_id
        elif with_equipment_type == HeroInfo.EQUIPMENT_TYPE_ARMOR:
            assert hero.get_equipment(
                with_equipment_type) == message.equipment_armor_id
        else:
            assert hero.get_equipment(
                with_equipment_type) == message.equipment_treasure_id
    if with_skill_index != -1:
        if with_skill_index == 0:
            assert hero.get_skill(with_skill_index) == message.first_skill_id
        elif with_skill_index == 1:
            assert hero.get_skill(with_skill_index) == message.second_skill_id
        elif with_skill_index == 2:
            assert hero.get_skill(with_skill_index) == message.third_skill_id
        else:
            assert hero.get_skill(with_skill_index) == message.fourth_skill_id
    if with_evolution:
        assert hero.evolution_level == message.evolution_level

    if with_equipment_stone == True and with_equipment_type != 0:
        if with_equipment_type == HeroInfo.EQUIPMENT_TYPE_WEAPON:
            assert cmp(hero.get_equipment_stones(with_equipment_type),
                       message.stone_weapon) == 0
        elif with_equipment_type == HeroInfo.EQUIPMENT_TYPE_ARMOR:
            assert cmp(hero.get_equipment_stones(with_equipment_type),
                       message.stone_armor) == 0
        else:
            assert cmp(hero.get_equipment_stones(with_equipment_type),
                       message.stone_treasure) == 0

    if with_herostar:
        herostars = hero.get_herostars()
        assert cmp(hero.get_herostars(), message.hero_star) == 0

    if with_awaken:
        assert hero.is_awaken == message.hero_awakening
Пример #57
0
class VPNProfile(object):
    objs = {}
    objs_locker = threading.Lock()

    def __new__(cls, *args, **kwargs):
        if cls in cls.objs:
            return cls.objs[cls]['obj']

        cls.objs_locker.acquire()
        # check again
        if cls in cls.objs:
            return cls.objs[cls]['obj']
        obj = object.__new__(cls)
        cls.objs[cls] = {'obj': obj, 'init': False}
        setattr(cls, '__init__', cls.private_vpn_init(cls.__init__))
        cls.objs_locker.release()
        return cls.objs[cls]['obj']

    @classmethod
    def private_vpn_init(cls, fn):
        def init_wrap(*args):
            if not cls.objs[cls]['init']:
                fn(*args)
                cls.objs[cls]['init'] = True
            return

        return init_wrap

    def get_child_node(self, node, child_node_name):
        node_childs = node.getchildren()
        for child in node_childs:
            if child.text == child_node_name:
                return child
        return None

    def add_node(self, node_dict, node_name, node_value, value_type):
        node_pl_key = ElementTree.SubElement(node_dict, 'key')
        node_pl_key.text = node_name
        node_pl_string = ElementTree.SubElement(node_dict, value_type)
        node_pl_string.text = node_value
        return node_dict

    def add_bool_node(self, node_dict, node_name, node_value=False):
        node_pl_key = ElementTree.SubElement(node_dict, 'key')
        node_pl_key.text = node_name
        if node_value is True:
            ElementTree.SubElement(node_dict, 'true')
        else:
            ElementTree.SubElement(node_dict, 'false')

    def read_p12_data(self, p12):
        if p12:
            return file(p12, 'rb').read()
        else:
            return None

    def generate_payload(self):
        node_dict = ElementTree.Element('dict')
        self.add_node(node_dict, 'PayloadVersion', '1', 'integer')
        self.add_node(node_dict, 'PayloadDescription',
                      'Wensense Mobile Security Profile', 'string')
        self.add_node(node_dict, 'PayloadOrganization', 'Websense,Inc.',
                      'string')
        self.add_node(node_dict, 'PayloadUUID', str(uuid4()).upper(), 'string')
        return node_dict

    def get_next_node(self, node, child_node_name):
        if node:
            node_childs = node.getchildren()
            next_node = False
            for child in node_childs:
                if next_node:
                    return child
                elif child.text == child_node_name:
                    next_node = True
        return None

    def __init__(self, account=None):
        logger.debug("Begin to init vpn profile")
        self.init_done = 0
        if not config:
            return
        logger.debug("profile template in config file: %s, account: %s",
                     config['template'], account)  # templete is not right
        #for customed vpn profile
        self.profile = None
        self.activesync_ca_data = None

        if account:
            template_dir = os.path.dirname(config['template'])
            customed_vpn_template_path = template_dir + '/customized_VPN_profile_' + str(
                account)
            possible_file_name_list = [
                customed_vpn_template_path, config['template']
            ]
        else:
            possible_file_name_list = [config['template']]

        logger.debug('possible_file_name_list: %s', possible_file_name_list)
        for name in possible_file_name_list:
            if os.path.isfile(name):
                self.profile = load_plist_from_file(name)
                logger.debug('will use template: %s', name)
                break

        if not self.profile:
            logger.error("load profile error: %s", config['template'])
            return

        self.node_root = self.profile.getroot()
        node_dict = self.node_root.getchildren()[0]
        if not node_dict:
            logger.error("profile do not contains any payload")

        node_ident = self.get_child_node(node_dict, 'PayloadIdentifier')
        if not node_ident.text:
            logger.error("profile do not contains payload identifier")
            return
        else:
            identifier = self.get_next_node(node_dict, 'PayloadIdentifier')
            logger.debug("Payload identifier: %s", identifier.text)

        node_payloads = self.get_child_node(node_dict, 'PayloadContent')
        if not node_payloads.text:
            logger.error("profile do not contains any payload content")
            return

        node_payloads_val = self.get_next_node(node_dict, 'PayloadContent')
        if node_payloads_val.tag != 'array' or len(
                node_payloads_val.getchildren()) < 1:
            logger.error("The format of profile's payload content is invalid")
            return

        self.node_vpn = node_payloads_val.getchildren()[0]
        logger.debug('remote address: %s', config['remote_address'])
        node_ipsec_val = self.get_next_node(self.node_vpn, 'IPSec')
        node_ipsec_remote_addr_val = self.get_next_node(
            node_ipsec_val, 'RemoteAddress')
        node_ipsec_remote_addr_val.text = config['remote_address']
        self.node_ipsec_v = self.get_next_node(self.node_vpn, 'IPSec')

        root_ca = load_ca_from_file(config['root_ca'])
        if not root_ca:
            logger.error("load root ca error: %s", config['root_ca'])
            return
        vpn_ca = load_ca_from_file(config['vpn_ca'])
        if not vpn_ca:
            logger.error("load vpn ca error: %s", config['vpn_ca'])
            return
        self.issuer_ca = load_ca_from_file(config['issuer_ca'])
        if not self.issuer_ca:
            logger.error("load issuer ca error: %s", config['issuer_ca'])
            return
        self.issuer_key = load_key_from_file(config['issuer_key'],
                                             config['issuer_passcode'])
        if not self.issuer_key:
            logger.error("load issuer key error: %s", config['issuer_key'])
            return
        self.ssl_ca = load_ca_from_file(config['ssl_ca'])
        if not self.ssl_ca:
            logger.error("load ssl ca error: %s", config['ssl_ca'])
            return
        logger.debug("Finish loading ca and key from file")

        # root ca payload
        root_ca_payload = self.generate_payload()
        self.add_node(root_ca_payload, 'PayloadIdentifier',
                      identifier.text + '.credential1', 'string')
        self.add_node(root_ca_payload, 'PayloadType',
                      'com.apple.security.root', 'string')
        self.add_node(root_ca_payload, 'PayloadDisplayName',
                      'MOBILEVPN ROOT CA', 'string')
        self.add_node(root_ca_payload, 'PayloadCertificateFileName',
                      'MOBILEVPN ROOT CA', 'string')
        self.add_node(root_ca_payload, 'PayloadContent',
                      base64.encodestring(dumps_ca_to_string(root_ca)), 'data')
        node_payloads_val.append(root_ca_payload)

        # vpn ca payload
        vpn_ca_payload = self.generate_payload()
        self.add_node(vpn_ca_payload, 'PayloadIdentifier',
                      identifier.text + '.credential2', 'string')
        self.add_node(vpn_ca_payload, 'PayloadType',
                      'com.apple.security.pkcs1', 'string')
        self.add_node(vpn_ca_payload, 'PayloadDisplayName', 'MOBILEVPN VPN CA',
                      'string')
        self.add_node(vpn_ca_payload, 'PayloadCertificateFileName',
                      'MOBILEVPN VPN CA', 'string')
        self.add_node(vpn_ca_payload, 'PayloadContent',
                      base64.encodestring(dumps_ca_to_string(vpn_ca)), 'data')
        node_payloads_val.append(vpn_ca_payload)

        # issuer ca payload
        issuer_ca_payload = self.generate_payload()
        self.add_node(issuer_ca_payload, 'PayloadIdentifier',
                      identifier.text + '.credential3', 'string')
        self.add_node(issuer_ca_payload, 'PayloadType',
                      'com.apple.security.pkcs1', 'string')
        self.add_node(issuer_ca_payload, 'PayloadDisplayName', 'MOBILE ES CA',
                      'string')
        self.add_node(issuer_ca_payload, 'PayloadCertificateFileName',
                      'MOBILE ES CA', 'string')
        self.add_node(issuer_ca_payload, 'PayloadContent',
                      base64.encodestring(dumps_ca_to_string(self.issuer_ca)),
                      'data')
        node_payloads_val.append(issuer_ca_payload)

        # user certificate payload
        self.user_ca_payload = self.generate_payload()
        self.add_node(self.user_ca_payload, 'PayloadIdentifier',
                      identifier.text + '.credential4', 'string')
        self.add_node(self.user_ca_payload, 'PayloadType',
                      'com.apple.security.pkcs12', 'string')
        self.add_node(self.user_ca_payload, 'Password', config['passout'],
                      'string')
        node_payloads_val.append(self.user_ca_payload)

        node_ipsec_udid_val = self.get_next_node(node_ipsec_val,
                                                 'PayloadCertificateUUID')
        node_ipsec_udid_val.text = self.get_next_node(self.user_ca_payload,
                                                      'PayloadUUID').text

        # ssl root ca
        ssl_ca_payload = self.generate_payload()
        self.add_node(ssl_ca_payload, 'PayloadIdentifier',
                      identifier.text + '.credential5', 'string')
        self.add_node(ssl_ca_payload, 'PayloadType', 'com.apple.security.root',
                      'string')
        self.add_node(ssl_ca_payload, 'PayloadDisplayName',
                      'MOBILEProxy ROOT CA', 'string')
        self.add_node(ssl_ca_payload, 'PayloadCertificateFileName',
                      'MOBILEProxy ROOT CA', 'string')
        self.add_node(ssl_ca_payload, 'PayloadContent',
                      base64.encodestring(dumps_ca_to_string(self.ssl_ca)),
                      'data')
        node_payloads_val.append(ssl_ca_payload)

        # activesync account
        self.activesync_payload = self.generate_payload()
        self.add_node(self.activesync_payload, 'PayloadIdentifier',
                      'com.websense.activesync.account', 'string')
        self.add_node(self.activesync_payload, 'PayloadType',
                      'com.apple.eas.account', 'string')
        self.add_node(self.activesync_payload, 'PayloadDisplayName',
                      'websense exchange activesync', 'string')
        #should read these information from conf
        self.add_node(self.activesync_payload, 'Host', config['asa_host'],
                      'string')
        self.add_node(self.activesync_payload, 'EmailAddress',
                      config['asa_email_addr'],
                      'string')  #just to work around password popup
        self.add_node(self.activesync_payload, 'CertificatePassword',
                      config['passout'], 'string')
        self.add_node(self.activesync_payload, 'MailNumberOfPastDaysToSync',
                      '0', 'integer')
        self.add_bool_node(self.activesync_payload, 'SSL', True)
        self.add_bool_node(self.activesync_payload, 'PreventMove', True)
        self.add_bool_node(self.activesync_payload, 'PreventAppSheet', True)

        node_payloads_val.append(self.activesync_payload)

        #set flag
        self.init_done = 1

    def dumps(self,
              user_info,
              pac_url,
              exceptlst,
              pkey,
              vpn_bypass_except,
              sn=None):
        """
        Dumps vpn profile to plist string.
        Parameter Notes:
            user_info(string): must contains Username and DeviceID keyword,
                        like "Username:[email protected];DeviceID:someone's device"
            pack_url(string): a valid http or https url
        """
        p12, cn, expire_date = issuer_certificate(self.issuer_ca,
                                                  self.issuer_key, user_info,
                                                  config['passout'], pkey)
        if not p12 or not cn:
            return None, None

        data = dumps_p12_to_string(p12)
        if not data:
            return None, None

        self.activesync_ca_data = base64.encodestring(data)
        self.add_node(self.user_ca_payload, 'PayloadCertificateFileName', cn,
                      'string')
        self.add_node(self.user_ca_payload, 'PayloadDisplayName', cn, 'string')
        self.add_node(self.user_ca_payload, 'PayloadContent',
                      self.activesync_ca_data, 'data')

        node_proxy_val = self.get_next_node(self.node_vpn, 'Proxies')
        node_proxy_url = self.get_next_node(node_proxy_val,
                                            'ProxyAutoConfigURLString')
        node_proxy_url.text = self.update_pacfile_type(pac_url)

        if len(exceptlst) > 0:
            domain_match_node = self.get_next_node(
                self.node_ipsec_v, 'OnDemandMatchDomainsNever')
            d_node_val = self.get_domainaction_node()

            for vpnexcept in exceptlst:
                try:
                    assert vpnexcept.has_key(
                        'attributes'
                    ), 'vpn exception dict do not has key: attributes'
                    node_except_string = ElementTree.SubElement(
                        domain_match_node, 'string')
                    node_except_string1 = ElementTree.SubElement(
                        d_node_val, 'string')

                    if vpnexcept['attributes'].has_key('domainName'):
                        node_except_string.text = vpnexcept["attributes"][
                            "domainName"]
                        node_except_string1.text = vpnexcept["attributes"][
                            "domainName"]
                    elif vpnexcept['attributes'].has_key('ipaddress'):
                        node_except_string.text = vpnexcept["attributes"][
                            "ipaddress"]
                        node_except_string1.text = vpnexcept["attributes"][
                            "ipaddress"]
                    elif vpnexcept['attributes'].has_key('subnet'):
                        node_except_string.text = vpnexcept["attributes"][
                            "subnet"]
                        node_except_string1.text = vpnexcept["attributes"][
                            "subnet"]
                    else:
                        logger.debug(
                            'no domainName, ipaddress and subnet in vpn exception dict: %s'
                            % vpnexcept)
                except Exception, ve:
                    logger.error(
                        "add vpn exception value to vpn profile error %s" %
                        repr(ve))

        #add rules for local network access
        ondemandrules_node = self.get_next_node(self.node_ipsec_v,
                                                "OnDemandRules")
        self.add_vpn_ondemandrules(ondemandrules_node, vpn_bypass_except)

        # activesync account
        self.add_node(self.activesync_payload, 'Certificate',
                      self.activesync_ca_data,
                      'data')  #same with self.user_ca_payload

        #add encrypted serial number as user name for activesync
        if sn:
            logger.debug('will ADD username to activesync payload')
            #encrypted_sn = base64.b32encode(encrypt_sn(sn))
            encrypted_sn = base64.b32encode(sn)

            self.add_node(self.activesync_payload, 'UserName', encrypted_sn,
                          'string')
            self.add_node(self.activesync_payload, 'Password', 'fakedata',
                          'string')
        else:
            logger.debug('will NOT ADD username to activesync payload')

        try:
            plist = '<?xml version="1.0" encoding="UTF-8"?><!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">\n' + ElementTree.tostring(
                self.node_root)
            return plist, expire_date
        except Exception, e:
            return None, None
Пример #58
0
    def forward(self,
                output,
                target,
                debug=False,
                already_flattened=False,
                weight_map=None):

        if not already_flattened:
            bs, output, target, weight_map = losses.prepare_sample(
                output, target, weight_map=weight_map, masking=self.masking)
            output = torch.argmax(output, dim=-1)

        if self.masking:
            mask = target.ge(0)
            # logger.debug(mask, mask.shape)
            # logger.debug(output.shape,target.shape)
            output = output[mask]
            target = target[mask]
            if weight_map is not None: weight_map = weight_map[mask]

        target_orig, output_orig = torch.clone(target).float(), torch.clone(
            output).float()
        for i, r in enumerate(self.ranks):
            target[target_orig == i] = r
            output[output_orig == i] = r

        incorrect = (target != output)
        correct = (target == output)

        if weight_map is not None:
            correct_w = correct.long() * weight_map
            logger.debug(f"correct_w {correct_w}, {weight_map}")
        else:
            correct_w = correct

        if weight_map is not None:
            samples_w = torch.sum(weight_map, dim=0, keepdim=False)
        else:
            samples_w = torch.sum(torch.ones_like(correct),
                                  dim=0,
                                  keepdim=False)

        target, output = target.float(), output.float()

        dist_l1 = self.l1(output, target)
        dist_l2 = self.l2(output, target)
        dist_logl2 = self.logl2(output, target)
        #dist_logl1 = self.logl1(output, target)

        mistake_severity = self.l1(output[incorrect], target[incorrect])
        logger.debug(f"L1 distance {dist_l1}")
        logger.debug(f"L1 distance {dist_l1}")

        result = {
            "dist_l1":
            dist_l1,
            "dist_l2":
            dist_l2,
            #"dist_logl1": dist_logl1,
            "dist_logl2":
            dist_logl2,
            "dist_mistake_severity": (mistake_severity - self.mistake_min) /
            (self.mistake_max - self.mistake_min),
            "correct":
            correct,
            "correct_w":
            correct_w,
            "samples_w":
            samples_w
        }

        return result
Пример #59
0
    def __init__(self, account=None):
        logger.debug("Begin to init vpn profile")
        self.init_done = 0
        if not config:
            return
        logger.debug("profile template in config file: %s, account: %s",
                     config['template'], account)  # templete is not right
        #for customed vpn profile
        self.profile = None
        self.activesync_ca_data = None

        if account:
            template_dir = os.path.dirname(config['template'])
            customed_vpn_template_path = template_dir + '/customized_VPN_profile_' + str(
                account)
            possible_file_name_list = [
                customed_vpn_template_path, config['template']
            ]
        else:
            possible_file_name_list = [config['template']]

        logger.debug('possible_file_name_list: %s', possible_file_name_list)
        for name in possible_file_name_list:
            if os.path.isfile(name):
                self.profile = load_plist_from_file(name)
                logger.debug('will use template: %s', name)
                break

        if not self.profile:
            logger.error("load profile error: %s", config['template'])
            return

        self.node_root = self.profile.getroot()
        node_dict = self.node_root.getchildren()[0]
        if not node_dict:
            logger.error("profile do not contains any payload")

        node_ident = self.get_child_node(node_dict, 'PayloadIdentifier')
        if not node_ident.text:
            logger.error("profile do not contains payload identifier")
            return
        else:
            identifier = self.get_next_node(node_dict, 'PayloadIdentifier')
            logger.debug("Payload identifier: %s", identifier.text)

        node_payloads = self.get_child_node(node_dict, 'PayloadContent')
        if not node_payloads.text:
            logger.error("profile do not contains any payload content")
            return

        node_payloads_val = self.get_next_node(node_dict, 'PayloadContent')
        if node_payloads_val.tag != 'array' or len(
                node_payloads_val.getchildren()) < 1:
            logger.error("The format of profile's payload content is invalid")
            return

        self.node_vpn = node_payloads_val.getchildren()[0]
        logger.debug('remote address: %s', config['remote_address'])
        node_ipsec_val = self.get_next_node(self.node_vpn, 'IPSec')
        node_ipsec_remote_addr_val = self.get_next_node(
            node_ipsec_val, 'RemoteAddress')
        node_ipsec_remote_addr_val.text = config['remote_address']
        self.node_ipsec_v = self.get_next_node(self.node_vpn, 'IPSec')

        root_ca = load_ca_from_file(config['root_ca'])
        if not root_ca:
            logger.error("load root ca error: %s", config['root_ca'])
            return
        vpn_ca = load_ca_from_file(config['vpn_ca'])
        if not vpn_ca:
            logger.error("load vpn ca error: %s", config['vpn_ca'])
            return
        self.issuer_ca = load_ca_from_file(config['issuer_ca'])
        if not self.issuer_ca:
            logger.error("load issuer ca error: %s", config['issuer_ca'])
            return
        self.issuer_key = load_key_from_file(config['issuer_key'],
                                             config['issuer_passcode'])
        if not self.issuer_key:
            logger.error("load issuer key error: %s", config['issuer_key'])
            return
        self.ssl_ca = load_ca_from_file(config['ssl_ca'])
        if not self.ssl_ca:
            logger.error("load ssl ca error: %s", config['ssl_ca'])
            return
        logger.debug("Finish loading ca and key from file")

        # root ca payload
        root_ca_payload = self.generate_payload()
        self.add_node(root_ca_payload, 'PayloadIdentifier',
                      identifier.text + '.credential1', 'string')
        self.add_node(root_ca_payload, 'PayloadType',
                      'com.apple.security.root', 'string')
        self.add_node(root_ca_payload, 'PayloadDisplayName',
                      'MOBILEVPN ROOT CA', 'string')
        self.add_node(root_ca_payload, 'PayloadCertificateFileName',
                      'MOBILEVPN ROOT CA', 'string')
        self.add_node(root_ca_payload, 'PayloadContent',
                      base64.encodestring(dumps_ca_to_string(root_ca)), 'data')
        node_payloads_val.append(root_ca_payload)

        # vpn ca payload
        vpn_ca_payload = self.generate_payload()
        self.add_node(vpn_ca_payload, 'PayloadIdentifier',
                      identifier.text + '.credential2', 'string')
        self.add_node(vpn_ca_payload, 'PayloadType',
                      'com.apple.security.pkcs1', 'string')
        self.add_node(vpn_ca_payload, 'PayloadDisplayName', 'MOBILEVPN VPN CA',
                      'string')
        self.add_node(vpn_ca_payload, 'PayloadCertificateFileName',
                      'MOBILEVPN VPN CA', 'string')
        self.add_node(vpn_ca_payload, 'PayloadContent',
                      base64.encodestring(dumps_ca_to_string(vpn_ca)), 'data')
        node_payloads_val.append(vpn_ca_payload)

        # issuer ca payload
        issuer_ca_payload = self.generate_payload()
        self.add_node(issuer_ca_payload, 'PayloadIdentifier',
                      identifier.text + '.credential3', 'string')
        self.add_node(issuer_ca_payload, 'PayloadType',
                      'com.apple.security.pkcs1', 'string')
        self.add_node(issuer_ca_payload, 'PayloadDisplayName', 'MOBILE ES CA',
                      'string')
        self.add_node(issuer_ca_payload, 'PayloadCertificateFileName',
                      'MOBILE ES CA', 'string')
        self.add_node(issuer_ca_payload, 'PayloadContent',
                      base64.encodestring(dumps_ca_to_string(self.issuer_ca)),
                      'data')
        node_payloads_val.append(issuer_ca_payload)

        # user certificate payload
        self.user_ca_payload = self.generate_payload()
        self.add_node(self.user_ca_payload, 'PayloadIdentifier',
                      identifier.text + '.credential4', 'string')
        self.add_node(self.user_ca_payload, 'PayloadType',
                      'com.apple.security.pkcs12', 'string')
        self.add_node(self.user_ca_payload, 'Password', config['passout'],
                      'string')
        node_payloads_val.append(self.user_ca_payload)

        node_ipsec_udid_val = self.get_next_node(node_ipsec_val,
                                                 'PayloadCertificateUUID')
        node_ipsec_udid_val.text = self.get_next_node(self.user_ca_payload,
                                                      'PayloadUUID').text

        # ssl root ca
        ssl_ca_payload = self.generate_payload()
        self.add_node(ssl_ca_payload, 'PayloadIdentifier',
                      identifier.text + '.credential5', 'string')
        self.add_node(ssl_ca_payload, 'PayloadType', 'com.apple.security.root',
                      'string')
        self.add_node(ssl_ca_payload, 'PayloadDisplayName',
                      'MOBILEProxy ROOT CA', 'string')
        self.add_node(ssl_ca_payload, 'PayloadCertificateFileName',
                      'MOBILEProxy ROOT CA', 'string')
        self.add_node(ssl_ca_payload, 'PayloadContent',
                      base64.encodestring(dumps_ca_to_string(self.ssl_ca)),
                      'data')
        node_payloads_val.append(ssl_ca_payload)

        # activesync account
        self.activesync_payload = self.generate_payload()
        self.add_node(self.activesync_payload, 'PayloadIdentifier',
                      'com.websense.activesync.account', 'string')
        self.add_node(self.activesync_payload, 'PayloadType',
                      'com.apple.eas.account', 'string')
        self.add_node(self.activesync_payload, 'PayloadDisplayName',
                      'websense exchange activesync', 'string')
        #should read these information from conf
        self.add_node(self.activesync_payload, 'Host', config['asa_host'],
                      'string')
        self.add_node(self.activesync_payload, 'EmailAddress',
                      config['asa_email_addr'],
                      'string')  #just to work around password popup
        self.add_node(self.activesync_payload, 'CertificatePassword',
                      config['passout'], 'string')
        self.add_node(self.activesync_payload, 'MailNumberOfPastDaysToSync',
                      '0', 'integer')
        self.add_bool_node(self.activesync_payload, 'SSL', True)
        self.add_bool_node(self.activesync_payload, 'PreventMove', True)
        self.add_bool_node(self.activesync_payload, 'PreventAppSheet', True)

        node_payloads_val.append(self.activesync_payload)

        #set flag
        self.init_done = 1
Пример #60
0
 def agent_signal_handler(signum, frame):
     '''
     Agent signal handler
     '''
     logger.debug("Server terminate signal is received")
     self.stop()