def post(url, endpoint, payload): """ Make a POST request to the desired URL with the supplied Payload :param url: URL to make the POST Request to :type url: str :param endpoint: Path of the URL to make the request to :type endpoint: str :param payload: POST Payload :type payload: dict :return: Result JSON, If the request was made successfully :rtype: str, bool """ url_endpoint = url + endpoint Logging.get_logger(__name__).info('Making POST Request to ' + url_endpoint) desk_config = Config.load_params()['Desk.com'] if desk_config['auth_method'] == 'oauth': oauth_session = OAuth1Session(desk_config['client_key'], client_secret=desk_config['client_secret'], resource_owner_key=desk_config['resource_owner_key'], resource_owner_secret=desk_config['resource_owner_secret']) request = oauth_session.post(url_endpoint, payload) else: request = requests.post(url_endpoint, json=payload, auth=(desk_config['username'], desk_config['password'])) try: return request.json(), request.status_code == 200 except Exception, e: Logging.get_logger(__name__).error('Problem Getting JSON from POST Request - %s' % e.message) return []
def get(url, endpoint): """ Make a GET request to the desk.com API using the pre-configured authentication method :param url: URL to make the GET Request to :type url: str :param endpoint: Path of the URL to make the request to :type endpoint: str :return: Returned JSON :rtype: str """ url_endpoint = url + endpoint Logging.get_logger(__name__).info('Making GET Request to ' + url_endpoint) desk_config = Config.load_params()['Desk.com'] if desk_config['auth_method'] == 'oauth': oauth_session = OAuth1Session(desk_config['client_key'], client_secret=desk_config['client_secret'], resource_owner_key=desk_config['resource_owner_key'], resource_owner_secret=desk_config['resource_owner_secret']) request = oauth_session.get(url_endpoint) else: request = requests.get(url_endpoint, auth=(desk_config['username'], desk_config['password'])) try: return request.json(), request.status_code == 200 except Exception, e: Logging.get_logger(__name__).error('Problem Getting JSON from GET Request - %s' % e.message) return []
def create(self): conn = libvirt.open(DC.get("virsh", "connect")) try: storagePool = conn.storagePoolLookupByName(self.pool) storagePool.refresh(0) except: Logging.errorExit("There is no '%s' libvirt storage pool." % self.pool) path = os.path.basename(self.path) size = Units(self.size).convertTo("B") diskXML = """ <volume> <name>%s</name> <capacity>%i</capacity> <allocation>0</allocation> <target> <format type='raw'/> </target> </volume> """ diskXML = diskXML % (path, size) try: storage = storagePool.createXML(diskXML, 0) except Exception as e: Logging.errorExit("%s (name: %s)" % (str(e), path)) return storage
class GunControl(object): def __init__(self, cont): self.log = Logging("GunControl", "Debug") self.cont = cont self.own = cont.owner self.scene = bge.logic.getCurrentScene() self.bullets = self.own["Bullets"] if self.own["Bullets"] else 0 self.magazines = self.own["Magazines"] if self.own["Magazines"] else 0 self.reload = self.cont.sensors["Reload"] self.shoot = self.cont.sensors["Shoot"] self.aim = self.cont.sensors["Ray"] self.triggerPull = self.cont.sensors["Trigger"] self.gun = self.own.parent self._updateHUD() self.log.msg("Init Completed.") def update(self): if self.aim.positive: # Add the laser targeting pointer to the scene bge.render.drawLine(self.own.worldPosition, self.aim.hitPosition, [255, 0, 0]) if self.reload.positive and self.magazines > 0: self._reload() self.gun.state = 1 # flip the gun back to shoot mode if self.bullets > 0: if self.shoot.positive or self.triggerPull.positive: self._shoot() else: self.gun.state = 2 # gun is empty self._updateHUD() def _shoot(self): self.bullets += -1 if self.aim.positive: bulletForce = self.scene.addObject("BulletForce", self.own, 3) bulletForce.worldPosition = Vector(self.aim.hitPosition) + (Vector(self.aim.hitNormal) * 0.01) bulletHole = self.scene.addObject("BulletHole", self.own, 200) # position the bullet based on the ray, give a margin factor due to rendering collision bulletHole.worldPosition = Vector(self.aim.hitPosition) + (Vector(self.aim.hitNormal) * 0.01) bulletHole.alignAxisToVect(self.aim.hitNormal, 2) self._activate(self.cont.actuators["Fire"]) self._activate(self.cont.actuators["MuzzleFlash"]) def _reload(self): # self._activate(self.cont.actuators["Reload"]) self.magazines += -1 self.bullets = self.own["Bullets"] if self.own["Bullets"] else 0 def _updateHUD(self): bge.logic.sendMessage("Ammo", str(self.bullets)) bge.logic.sendMessage("Clips", str(self.magazines)) if self.magazines == 0 and self.bullets == 0: bge.logic.sendMessage("GameOver", "GameOver") def _activate(self, actuator): self.cont.activate(actuator)
class Parsers: """ parse json file, store important stuff get no of pages-> pages page_counter <- 1 while(page_counter < pages): get_page(page_counter) for recording in recordings: get_recording_details() get_audio_file() page_counter++ """ MAX_IMAGES_URL = 4 MAX_NO_THREADS = 100 CONFIG_FILE = 'bsrs.cfg' BIRD_SOUNDS_DIR = 'BirdSounds/' def __init__(self): self.fetcher = Fetcher() self.logger = Logging() self.config_file = Parsers.CONFIG_FILE self.config = ConfigParser() self.load_config_file() creds = self.get_db_creds() self.database = MySQLDatabases(hostname=creds['hostname'], username=creds['username'], password=creds['passwd'], database=creds['db_name']) #queues self.birdID_queue = self.wavFile_queue = Queue() self.soundtype_queue = self.soundURL_queue = Queue() def get_db_creds(self): """ load db creds from config file """ hostname = self.config.get('database', 'db_hostname') username = self.config.get('database', 'db_username') passwd = self.config.get('database', 'db_password') db_name = self.config.get('database', 'db_dbname') return {'hostname': hostname, 'username': username, 'passwd': passwd, 'db_name': db_name} def load_config_file(self): """ Load config file """ try: self.config.read(self.config_file) info_msg = "Loaded config file %s" % self.config_file self.logger.write_log('fetcher', 'i', info_msg) except Exception, e: info_msg = "config file %s missing" % self.config_file self.logger.write_log('fetcher', 'e', info_msg) raise Exception(info_msg)
def create(self): Logging.info("Creating the network ...") conn = libvirt.open(DC.get("virsh", "connect")) configFile = file(self.config, "r") #print configFile.read() self.network = conn.networkDefineXML(configFile.read()) self.network.setAutostart(1) self.network.create() return
def get_filters(self): """ Get all the filters in Desk.com :return: All Filters in Desk, If the request was completed successfully. :rtype: str, bool """ results, ok = self.get(self.url, 'filters') Logging.get_logger(__name__).debug('Get Filters returned %s' % str(results)) return results, ok
def check(self): network = self.parseName() Logging.info("Checking network '%s'..." % network) conn = libvirt.open(DC.get("virsh", "connect")) networks = conn.listNetworks() if not network in networks: return True return False
class GunUdpHandler(object): def __init__(self): self.protocolFormat = '<ffffff' self.log = Logging("GunUDPHandler","Debug") self.log.msg("Init Completed.") def parse(self,rawData): try: data = struct.unpack(self.protocolFormat,rawData) #self.log.msg("data: "+str(data[0])+" "+str(data[1])) return str(data[0])+','+str(data[1])+','+str(data[2])+','+str(data[3])+','+str(data[4])+','+str(data[5]) except Exception as e: self.log.msg("error: "+str(e))
def resolve_case(self, case_id): """ Mark a Case as Resolved :param case_id: Case to Resolve :type case_id: int :return: The Updated Case, If the request was made successfully. :rtype: str, bool """ Logging.get_logger(__name__).info("Resolving Case with ID: %d" % case_id) result, ok = self.patch(self.url, 'cases/' + str(case_id), {"status": "resolved"}) return result, ok
def run(self): if self.configPath and self.config.sections(): self.start() self.login() self.setNetwork() output = self.getNetwork() self.stop() return output else: Logging.warning("Network settings not set. Skipping setting" " network on guest '%s'.", self.guest.getHostName()) return {}
def get_groups(self): """ Get all the Groups in Desk.com :return: All Groups in Desk.com, If the request was successful :rtype: str, bool """ results, ok = self.get(self.url, 'groups') if ok: results = results['_embedded']['entries'] Logging.get_logger(__name__).debug('Get Groups returned %s' % str(results)) return results, ok
def __init__(self, justPlots = False): self.__name__ = "Core" self.configManager = ConfigurationManager() # These return True of False depending on whether loading the conf was a success. # It should be checked if the conf was loaded successfully and failures should be logged. self.configManager.loadConf(CONFIG_CORE, True) self.configManager.loadConf(CONFIG_SETTINGS, True) self.configManager.loadConf(CONFIG_FORMS, True) self.configManager.loadConf(CONFIG_URLMAP, True) self.configManager.loadConf(CONFIG_MESSAGES, True) self.moduleManager = ModuleManager(self) self.settingsManager = SettingsManager(self) self.clientManager = ClientManager(self) self.sensorManager = SensorManager(self) self.deviceManager = DeviceManager(self) self.taskManager = TaskManager(self) self.messageManager = MessageManager(self) self.logging = Logging(self) if self.settingsManager.equals("plottype", "matplotlib"): from Plot import Plot self.plot = Plot(self) self.protocol = Protocol(self) if not justPlots: self.connection = Connection(self) if not justPlots: self.scheduler = Scheduler() if not justPlots: self.webServer = WebServer(self.connection.getLocalIP(), self.settingsManager.getValueByName("listenport")) # Currently binds to localhost. But this needs to be fixed so other connections can be listened to too.
def umount(self): if os.path.exists(self.mountPoint) and os.path.isdir(self.mountPoint): Logging.debug("Unmounting %s." % self.mountPoint) mountProcess = subprocess.Popen(["/bin/umount", self.mountPoint]) mountProcess.wait() if mountProcess.returncode == 0 and len(os.listdir(self.mountPoint)) == 0: if self.createdMountPoint: Logging.debug("Deleting mounting point: %s" % self.mountPoint) os.rmdir(self.mountPoint) self.createdMountPoint = False return True return False
def add_note_to_case(self, case_id, note): """ Add a note to the Supplied Case :param case_id: Case to add note to :type case_id: int :param note: Note to be added to Case :type note: str :return: The Updated Case, If the request was made successfully. :rtype: str, bool """ Logging.get_logger(__name__).info("Adding Note to Case with ID: %d" % case_id) result, ok = self.post(self.url, 'cases/' + str(case_id) + '/notes', {"body": note}) return result, ok
class GameOver(object): ''' this class is the main entry point and control of the game scene ''' def __init__(self,cont): self.cont = cont self.own = cont.owner self.scene = bge.logic.getCurrentScene() self.log = Logging("GameOver","Debug") self.showScoreboard = self.cont.actuators['GoScoreboard'] self.showScoreboard.scene="Scoreboard" self.kbd = self.cont.sensors['Keyboard'] self.log.msg('init completed') def update(self): if self.kbd.positive: self.cont.activate(self.showScoreboard)
def __init__(self): self.host = "0.0.0.0" self.port = 10001 self.protocol = ProtocolHandler() self.socketClient = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.socketClient.bind((self.host, self.port)) self.socketClient.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 256) self.socketClient.setblocking(0) self.log = Logging("Networking", "Debug") self.log.msg("Init Completed.")
def download(self): fileName = self.path.rsplit("/", 1) filePath = "%s/%i-%s" % (self.tempStorage, hash(self), fileName[1]) if os.path.exists(filePath): self.path = filePath self.downloaded = True return self.downloaded Logging.debug("Trying download %s to %s." % (self.path, filePath)) try: httpRequest = urllib2.urlopen(self.path) except urllib2.HTTPError as e: return None if (not os.path.exists(self.tempStorage) or not os.access(self.tempStorage, os.W_OK) or System.getFreeSpace(self.tempStorage) < int(httpRequest.info().get("Content-Length"))): return None try: iso = file(filePath, "w") while 1: buf = httpRequest.read(16*1024) if not buf: break iso.write(buf) iso.close() except IOError as e: return None self.path = filePath self.downloaded = os.path.exists(filePath) return self.downloaded
def __init__(self,cont): self.cont = cont self.own = cont.owner self.scene = bge.logic.getCurrentScene() self.log = Logging("GameOver","Debug") self.showScoreboard = self.cont.actuators['GoScoreboard'] self.showScoreboard.scene="Scoreboard" self.kbd = self.cont.sensors['Keyboard'] self.log.msg('init completed')
def __init__(self,cont): self.cont = cont self.own = cont.owner self.log = Logging("Splash","Debug") self.scene = bge.logic.getCurrentScene() self.title = self.scene.objects["Title"] self.title.text = "SAND-Matrix" self.log.msg("Init complete.")
def __init__(self,cont): self.cont = cont self.own = cont.owner self.scene = bge.logic.getCurrentScene() self.log = Logging("Menu","Debug") self.startGame = self.cont.actuators['StartGame'] self.startGame.scene="Main" self.mouse = self.cont.sensors['Mouse'] self.trigger = self.cont.sensors['Trigger'] self.log.msg('init completed')
class Splash(object): ''' this class is the Splash ''' def __init__(self,cont): self.cont = cont self.own = cont.owner self.log = Logging("Splash","Debug") self.scene = bge.logic.getCurrentScene() self.title = self.scene.objects["Title"] self.title.text = "SAND-Matrix" self.log.msg("Init complete.") def update(self): self._handleMessage() def _handleMessage(self): pass
class Networking(object): def __init__(self): self.host = "0.0.0.0" self.port = 10001 self.protocol = ProtocolHandler() self.socketClient = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) self.socketClient.bind((self.host, self.port)) self.socketClient.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 256) self.socketClient.setblocking(0) self.log = Logging("Networking", "Debug") self.log.msg("Init Completed.") def update(self): try: rawData, SRIP = self.socketClient.recvfrom(256) data = self.protocol.parse(rawData) self.log.msg(data) bge.logic.sendMessage("SpawnEnemy", data, "Source", "Source") except: pass
class Menu(object): ''' this class is the main entry point and control of the game scene ''' def __init__(self,cont): self.cont = cont self.own = cont.owner self.scene = bge.logic.getCurrentScene() self.log = Logging("Menu","Debug") self.startGame = self.cont.actuators['StartGame'] self.startGame.scene="Main" self.mouse = self.cont.sensors['Mouse'] self.trigger = self.cont.sensors['Trigger'] self.log.msg('init completed') def update(self): if self.mouse.positive or self.trigger.positive: self.log.msg('here') self.cont.activate(self.startGame)
def get_cases(self, case_filter=''): """ Get all cases that match the supplied filter. If no filter is supplied, all cases will be fetched. :param case_filter: Either Customer ID, Company ID, or Filter ID :type case_filter: str :return: All Cases that match the filter, If the request was completed successfully. :rtype: str, bool """ results, ok = self.get(self.url, 'cases' + case_filter) all_cases = results['_embedded']['entries'] Logging.get_logger(__name__).debug('Get Cases returned %s' % str(results)) while results['_links']['next'] is not None: results, ok = self.get(self.url, results['_links']['next']['href'].split('/')[-1]) Logging.get_logger(__name__).debug('Get Cases returned %s' % str(results)) all_cases.append(results['_embedded']['entries']) Logging.get_logger(__name__).info('Fetched a total of %d cases.' % len(all_cases)) marshaled_results = Marshal.desk_case(all_cases) for case in marshaled_results: case.customer = self.get_customer(case.customer_id)[0] return marshaled_results, ok
def install(self, handler=DefaultHandler): self.setValues() destroy = None servers = self.startHttpd() master, slave = pty.openpty() stdin = os.fdopen(master, "w") logPath = self.log % self.values Logging.debug("Trying write install log of VM '%s' to %s." % (self.guest.getHostName(), logPath)) stdout = open(logPath, "w") try: self.cmdInstall = self.cmdInstall.replace("\n", " ") Logging.info("Trying install guest '%s'..." % self.guest.getHostName()) Logging.debug(self.cmdInstall % self.values) process = subprocess.Popen(self.cmdInstall % self.values, shell=True, stdin=slave, stdout=stdout, stderr=stdout, close_fds=True) analyzator = handler(stdin, stdout, process) analyzator.handle() except InstallationError as e: destroy = subprocess.Popen(self.cmdDestroy % self.values, shell=True) Logging.info("Check installation log for details: %s" % logPath) raise e finally: if servers[0]: servers[0].kill() if servers[1]: servers[1].kill() if destroy: return not destroy.wait() Logging.info("Guest '%s' installed." % self.guest.getHostName()) return
def insertDNS(self, dnsDict): conn = libvirt.open(DC.get("virsh", "connect")) xmlDesc = self.network.XMLDesc(0) Logging.debug("Inserting DNS entries: %s" % str(dnsDict)) localDns = self.makeDNS(dnsDict) networkXml = xml.parseString(xmlDesc) dns = networkXml.getElementsByTagName("dns") if dns: dns = dns[0] for entry in localDns.getElementsByTagName("host"): dns.appendChild(entry) else: networkXml.documentElement.appendChild(localDns) self.network.destroy() self.network.undefine() self.network = conn.networkDefineXML(networkXml.documentElement.toxml()) self.network.setAutostart(1) self.network.create() values = {'addr': self.getIPv6Address(), 'prefix': self.getIPv6Prefix(), 'interface': self.getInterface()} if (DC.has_option("network", "manual-route") and DC.get("network", "manual-route", True)): cmd = DC.get("network", "manual-route", False, values) cmd = cmd.replace("\n", " ") subprocess.Popen(cmd, shell=True) else: Logging.warning("Manual route not set. Skipping.") return
class GunUdpListener(object): def __init__(self): self.host = "0.0.0.0" self.port = 10002 self.protocol = GunUdpHandler() self.socketClient = socket.socket(socket.AF_INET,socket.SOCK_DGRAM) self.socketClient.bind((self.host, self.port)) self.socketClient.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 256) self.socketClient.setblocking(0) self.log = Logging("GunUdpListener","Debug") self.log.msg("Init Completed.") def update(self): try: rawData, SRIP = self.socketClient.recvfrom(256) #self.log.msg(rawData) data = self.protocol.parse(rawData) #self.log.msg(data) bge.logic.sendMessage("GunPos", data) except Exception as e: #self.log.msg(e) pass
def __init__(self): self.fetcher = Fetcher() self.logger = Logging() self.config_file = Parsers.CONFIG_FILE self.config = ConfigParser() self.load_config_file() creds = self.get_db_creds() self.database = MySQLDatabases(hostname=creds['hostname'], username=creds['username'], password=creds['passwd'], database=creds['db_name']) #queues self.birdID_queue = self.wavFile_queue = Queue() self.soundtype_queue = self.soundURL_queue = Queue()
class ProtocolHandler(object): """ Struct-based Binary Protocol Definition (UDP) ============================================= < = little endian L = Source IP (integer) L = Destination IP (integer) """ def __init__(self): self.protocolFormat = "<LL" self.log = Logging("ProtocolHandler", "Debug") self.log.msg("Init Completed.") def parse(self, rawData): data = struct.unpack(self.protocolFormat, rawData) return str(data[0]) + "," + str(data[1]) def _dec2dot(numbericIP): if type(numbericIP) == types.StringType and not numbericIP.isdigit(): return None numIP = long(numbericIP) return "%d.%d.%d.%d" % ((numIP >> 24) & 0xFF, (numIP >> 16) & 0xFF, (numIP >> 8) & 0xFF, numIP & 0xFF)
def __init__(self) -> None: Logging.log('RegistrationManager initialized')
lr=conf.learning_rate, weight_decay=conf.weight_decay) scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.8) ############################## PREPARE DATASET ############################## print('System start to load data...') t0 = time() train_data, val_data, test_data = data_utils.load_all() t1 = time() print('Data has been loaded successfully, cost:%.4fs' % (t1 - t0)) ########################### FIRST TRAINING ##################################### check_dir('%s/train_%s_mrg_id_x.log' % (conf.out_path, conf.data_name)) log = Logging('%s/train_%s_mrg_id_06.py' % (conf.out_path, conf.data_name)) train_model_path = '%s/train_%s_mrg_id_06.mod' % (conf.out_path, conf.data_name) # prepare data for the training stage train_dataset = data_utils.TrainData(train_data) val_dataset = data_utils.TrainData(val_data) test_dataset = data_utils.TrainData(test_data) train_batch_sampler = data.BatchSampler(data.RandomSampler( range(train_dataset.length)), batch_size=conf.batch_size, drop_last=False) val_batch_sampler = data.BatchSampler(data.RandomSampler( range(val_dataset.length)), batch_size=conf.batch_size,
def __init__(self) -> None: Logging.log('Importer initialized')
def handleDeleteRequest(self, request: CSERequest, id: str, originator: str) -> Result: """ Handle a DELETE request. Delete the latest resource. """ Logging.logDebug('Deleting latest CIN from CNT') if (r := self._getLatest()) is None: return Result(rsc=RC.notFound, dbg='no instance for <latest>')
def shutdown(self) -> bool: Logging.log('GroupManager shut down') return True
model.cuda() #optimizer = torch.optim.SGD(model.parameters(), lr=conf.learning_rate, weight_decay=conf.weight_decay) optimizer = torch.optim.Adam(model.parameters(), lr=conf.learning_rate, weight_decay=conf.weight_decay) ############################## PREPARE DATASET ############################## print('System start to load data...') t0 = time() train_data, val_data, test_data = data_utils.load_all() t1 = time() print('Data has been loaded successfully, cost:%.4fs' % (t1 - t0)) ########################### FIRST TRAINING ##################################### check_dir('%s/train_%s_pmf_id_x.log' % (conf.out_path, conf.data_name)) log = Logging('%s/train_%s_ncf_id_X1.log' % (conf.out_path, conf.data_name)) train_model_path = '%s/train_%s_ncf_id_X1.mod' % (conf.out_path, conf.data_name) # prepare data for the training stage train_dataset = data_utils.TrainData(train_data) val_dataset = data_utils.TrainData(val_data) test_dataset = data_utils.TrainData(test_data) train_batch_sampler = data.BatchSampler(data.RandomSampler( range(train_dataset.length)), batch_size=conf.batch_size, drop_last=False) val_batch_sampler = data.BatchSampler(data.RandomSampler( range(val_dataset.length)), batch_size=conf.batch_size,
scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.8) ############################## PREPARE DATASET ############################## print('System start to load data...') t0 = time() train_data, val_data, test_data, \ train_user_historical_review_dict, train_item_historical_review_dict = data_utils.load_all() t1 = time() print('Data has been loaded successfully, cost:%.4fs' % (t1 - t0)) ########################### FIRST TRAINING ##################################### check_dir('%s/train_%s_aspect_rating_1_id_x.log' % (conf.out_path, conf.data_name)) log = Logging('%s/train_%s_aspect_rating_1_id_65.py' % (conf.out_path, conf.data_name)) train_model_path = '%s/train_%s_aspect_rating_1_id_65.mod' % ( conf.out_path, conf.data_name) # prepare data for the training stage train_dataset = data_utils.TrainData(train_data, train_user_historical_review_dict, train_item_historical_review_dict, train_data) val_dataset = data_utils.TrainData(val_data, train_user_historical_review_dict, train_item_historical_review_dict, train_data) test_dataset = data_utils.TrainData(test_data, train_user_historical_review_dict, train_item_historical_review_dict,
def _runNonBlockingRequestSync(self, request: CSERequest, reqRi: str) -> bool: """ Execute the actual request and store the result in the respective <request> resource. """ Logging.logDebug('Executing nonBlockingRequestSync') return self._executeOperation(request, reqRi).status
def __init__(self) -> None: self.enableTransit = Configuration.get('cse.enableTransitRequests') self.flexBlockingBlocking = Configuration.get( 'cse.flexBlockingPreference') == 'blocking' Logging.log('RequestManager initialized')
from fm import fm model = fm() #model.load_state_dict(torch.load('/content/drive/My Drive/task/aspect_based_rs/out/model/train_amazon_clothing_fm_id_2.mod')) model.cuda() optimizer = torch.optim.Adam(model.parameters(), lr=conf.lr, weight_decay=conf.weight_decay) scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.8) ########################### FIRST TRAINING ##################################### check_dir('%s/train_%s_fm_id_x.log' % (conf.out_path, conf.data_name)) log = Logging('%s/train_%s_fm_id_01.log' % (conf.out_path, conf.data_name)) train_model_path = '%s/train_%s_fm_id_01.mod' % (conf.out_path, conf.data_name) # prepare data for the training stage train_dataset = data_utils.TrainData(train_data) val_dataset = data_utils.TrainData(val_data) test_dataset = data_utils.TrainData(test_data) train_batch_sampler = data.BatchSampler(data.RandomSampler( range(train_dataset.length)), batch_size=conf.batch_size, drop_last=False) val_batch_sampler = data.BatchSampler(data.RandomSampler( range(val_dataset.length)), batch_size=conf.batch_size,
reqRi: str) -> bool: """ Execute the actual request and store the result in the respective <request> resource. """ Logging.logDebug('Executing nonBlockingRequestSync') return self._executeOperation(request, reqRi).status def _runNonBlockingRequestAsync(self, request: CSERequest, reqRi: str) -> bool: """ Execute the actual request and store the result in the respective <request> resource. In addition notify the notification targets. """ Logging.logDebug('Executing nonBlockingRequestAsync') if not (result := self._executeOperation(request, reqRi)).status: return False Logging.logDebug( 'Sending result notifications for nonBlockingRequestAsynch') # TODO move the notification to the notificationManager # The result contains the request resource (the one from the actual operation). # So we can just copy the individual attributes originator = result.resource['ors/fr'] responseNotification = { 'm2m:rsp': { 'rsc': result.resource['ors/rsc'], 'rqi': result.resource['ors/rqi'], 'pc': result.resource['ors/pc'], 'to': result.resource['ors/to'], 'fr': originator, 'rvi': request.headers.releaseVersionIndicator } }
class FCNT(AnnounceableResource): def __init__(self, dct: JSON = None, pi: str = None, fcntType: str = None, create: bool = False) -> None: super().__init__(T.FCNT, dct, pi, tpe=fcntType, create=create, attributePolicies=attributePolicies) self.resourceAttributePolicies = fcntPolicies # only the resource type's own policies if self.dict is not None: self.setAttribute('cs', 0, overwrite=False) # "current" attributes are added when necessary in the validate() method # Indicates whether this FC has flexContainerInstances. # Might change during the lifetime of a resource. Used for optimization self.hasInstances = False self.__validating = False self.ignoreAttributes = self.internalAttributes + [ 'acpi', 'cbs', 'cni', 'cnd', 'cs', 'cr', 'ct', 'et', 'lt', 'mbs', 'mia', 'mni', 'or', 'pi', 'ri', 'rn', 'st', 'ty', 'at' ] # Enable check for allowed sub-resources def canHaveChild(self, resource: Resource) -> bool: return super()._canHaveChild( resource, [ T.CNT, T.FCNT, T.SUB # FlexContainerInstances are added by the flexContainer itself ]) def activate(self, parentResource: Resource, originator: str) -> Result: if not (res := super().activate(parentResource, originator)).status: return res # register latest and oldest virtual resources Logging.logDebug( f'Registering latest and oldest virtual resources for: {self.ri}') if self.hasInstances: # add latest resource = Factory.resourceFromDict( {}, pi=self.ri, ty=T.FCNT_LA).resource # rn is assigned by resource itself if (res := CSE.dispatcher.createResource(resource)).resource is None: return Result(status=False, rsc=res.rsc, dbg=res.dbg) # add oldest resource = Factory.resourceFromDict( {}, pi=self.ri, ty=T.FCNT_OL).resource # rn is assigned by resource itself if (res := CSE.dispatcher.createResource(resource)).resource is None: return Result(status=False, rsc=res.rsc, dbg=res.dbg)
def _removeACP(self, srn: str, resource: Resource) -> Tuple[Resource, int, str]: """ Remove an ACP created during registration before. """ if (acpRes := CSE.dispatcher.retrieveResource(id=srn))[1] != C.rcOK: Logging.logWarn('Could not find ACP: %s' % srn) # ACP not found, either not created or already deleted
def shutdown(self) -> None: Logging.log('RegistrationManager shut down')
def start(conf, data, model, evaluate): log_dir = os.path.join(os.getcwd(), 'log') if not os.path.exists(log_dir): os.makedirs(log_dir) # define log name log_path = os.path.join( os.getcwd(), 'log/%s_%s.log' % (conf.data_name, conf.model_name)) # start to prepare data for training and evaluating data.initializeRankingHandle() d_train, d_val, d_test, d_test_eva = data.train, data.val, data.test, data.test_eva print('System start to load data...') t0 = time() d_train.initializeRankingTrain() d_val.initializeRankingVT() d_test.initializeRankingVT() d_test_eva.initalizeRankingEva() t1 = time() print('Data has been loaded successfully, cost:%.4fs' % (t1 - t0)) # prepare model necessary data. data_dict = d_train.prepareModelSupplement(model) model.inputSupply(data_dict) model.startConstructGraph() # standard tensorflow running environment initialize tf_conf = tf.ConfigProto() os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' tf_conf.gpu_options.allow_growth = True sess = tf.Session(config=tf_conf) sess.run(model.init) if conf.pretrain_flag == 1: model.saver.restore(sess, conf.pre_model) # set debug_flag=0, doesn't print any results log = Logging(log_path) print() log.record('Following will output the evaluation of the model:') # Start Training !!! for epoch in range(1, conf.epochs + 1): # optimize model with training data and compute train loss tmp_train_loss = [] t0 = time() #tmp_total_list = [] while d_train.terminal_flag: d_train.getTrainRankingBatch() d_train.linkedMap() train_feed_dict = {} for (key, value) in model.map_dict['train'].items(): train_feed_dict[key] = d_train.data_dict[value] [sub_train_loss, _] = sess.run(\ [model.map_dict['out']['train'], model.opt], feed_dict=train_feed_dict) tmp_train_loss.append(sub_train_loss) train_loss = np.mean(tmp_train_loss) t1 = time() # compute val loss and test loss d_val.getVTRankingOneBatch() d_val.linkedMap() val_feed_dict = {} for (key, value) in model.map_dict['val'].items(): val_feed_dict[key] = d_val.data_dict[value] val_loss = sess.run(model.map_dict['out']['val'], feed_dict=val_feed_dict) d_test.getVTRankingOneBatch() d_test.linkedMap() test_feed_dict = {} for (key, value) in model.map_dict['test'].items(): test_feed_dict[key] = d_test.data_dict[value] test_loss = sess.run(model.map_dict['out']['test'], feed_dict=test_feed_dict) t2 = time() # start evaluate model performance, hr and ndcg def getPositivePredictions(): d_test_eva.getEvaPositiveBatch() d_test_eva.linkedRankingEvaMap() eva_feed_dict = {} for (key, value) in model.map_dict['eva'].items(): eva_feed_dict[key] = d_test_eva.data_dict[value] positive_predictions = sess.run(model.map_dict['out']['eva'], feed_dict=eva_feed_dict) return positive_predictions def getNegativePredictions(): negative_predictions = {} terminal_flag = 1 while terminal_flag: batch_user_list, terminal_flag = d_test_eva.getEvaRankingBatch( ) d_test_eva.linkedRankingEvaMap() eva_feed_dict = {} for (key, value) in model.map_dict['eva'].items(): eva_feed_dict[key] = d_test_eva.data_dict[value] index = 0 tmp_negative_predictions = np.reshape( sess.run(model.map_dict['out']['eva'], feed_dict=eva_feed_dict), [-1, conf.num_evaluate]) for u in batch_user_list: negative_predictions[u] = tmp_negative_predictions[index] index = index + 1 return negative_predictions tt2 = time() index_dict = d_test_eva.eva_index_dict positive_predictions = getPositivePredictions() negative_predictions = getNegativePredictions() d_test_eva.index = 0 # !!!important, prepare for new batch hr, ndcg = evaluate.evaluateRankingPerformance(\ index_dict, positive_predictions, negative_predictions, conf.topk, conf.num_procs) tt3 = time() # print log to console and log_file log.record('Epoch:%d, compute loss cost:%.4fs, train loss:%.4f, val loss:%.4f, test loss:%.4f' % \ (epoch, (t2-t0), train_loss, val_loss, test_loss)) log.record('Evaluate cost:%.4fs, hr:%.4f, ndcg:%.4f' % ((tt3 - tt2), hr, ndcg)) ## reset train data pointer, and generate new negative data d_train.generateTrainNegative()
def shutdown(self) -> bool: Logging.log('RequestManager shut down') return True
review_optimizer = torch.optim.Adam(model.parameters(), lr=conf.learning_rate) rating_optimizer = torch.optim.Adam(model.parameters(), lr=conf.learning_rate, weight_decay=conf.weight_decay) ############################## PREPARE DATASET ############################## print('System start to load data...') t0 = time() train_data, val_data, test_data = data_utils.load_all() t1 = time() print('Data has been loaded successfully, cost:%.4fs' % (t1 - t0)) ########################### FIRST TRAINING ##################################### check_dir('%s/train_%s_lm_id_x.py' % (conf.out_path, conf.data_name)) log = Logging('%s/train_%s_mrg_id_X.py' % (conf.out_path, conf.data_name)) train_model_path = '%s/train_%s_mrg_id_X' % (conf.out_path, conf.data_name) # prepare data for the training stage train_dataset = data_utils.TrainData(train_data) train_batch_sampler = data.BatchSampler(data.RandomSampler(\ range(train_dataset.length)), batch_size=conf.batch_size, drop_last=False) val_dataset = data_utils.TrainData(val_data) val_batch_sampler = data.BatchSampler(data.SequentialSampler(\ range(val_dataset.length)), batch_size=conf.batch_size, drop_last=False) test_dataset = data_utils.TrainData(test_data) test_batch_sampler = data.BatchSampler(data.SequentialSampler(\ range(test_dataset.length)), batch_size=conf.batch_size, drop_last=False)
def __init__(self, message): if (not isinstance(message, "")) or message == "": self.message = u"帐号密码错误" else: self.message = message Logging.error(self.message)
def shutdown(self): Logging.log('GroupManager shut down')
return (CSEBase.CSEBase(jsn), C.rcOK) ######################################################################### # # Handling of Transit requests. Forward requests to the resp. remote CSE's. # # Forward a Retrieve request to a remote CSE def handleTransitRetrieveRequest(self, request, id, origin): if (url := self._getForwardURL(id)) is None: return (None, C.rcNotFound) if len(request.args) > 0: # pass on other arguments, for discovery url += '?' + urllib.parse.urlencode(request.args) Logging.log('Forwarding Retrieve/Discovery request to: %s' % url) return CSE.httpServer.sendRetrieveRequest(url, origin) # Forward a Create request to a remote CSE def handleTransitCreateRequest(self, request, id, origin, ty): if (url := self._getForwardURL(id)) is None: return (None, C.rcNotFound) Logging.log('Forwarding Create request to: %s' % url) return CSE.httpServer.sendCreateRequest(url, origin, data=request.data, ty=ty) # Forward a Update request to a remote CSE def handleTransitUpdateRequest(self, request, id, origin): if (url := self._getForwardURL(id)) is None: return (None, C.rcNotFound)
""" This is the Starting Point of Kijiji Web Scraping Script Environment Setup """ import sys from datetime import datetime from Logging import Logging from SetupEnvironment import SetupEnvironment """ Getting Basic Logging Options """ logger = Logging().get_logger("setup") if logger != None: # Checking Python Version python_major_version = sys.version_info.major if python_major_version < 3: logger.critical( "Setup Module : Version Issue : Please install Python Version 3 or greater to execute this script" ) else: logger.debug("Setup Module : Correct Python Version Found") logger.debug("Setup Module : Preparing Script Environment") # Staring Environment Setup setupEnvironment = SetupEnvironment(logger) setupEnvironment.installAndUnpgradeLibraries() logger.debug("Setup Module : Environment Setup Completed") else: print("Setup Module : Critical : Logging Setup Could Not Be Completed") print("Setup Module : Critical : Process will Exit") print("Setup Module : Critical : Contact Administrator For Resolution")
def _updateLocalCSR(self, localCSR, remoteCSE): Logging.logDebug('Updating local CSR: %s' % localCSR.rn) # copy attributes self._copyCSE2CSE(localCSR, remoteCSE) return CSE.dispatcher.updateResource(localCSR)
def __init__(self) -> None: # Add delete event handler because we like to monitor the resources in mid CSE.event.addHandler(CSE.event.deleteResource, self.handleDeleteEvent) # type: ignore Logging.log('GroupManager initialized')
def _deleteLocalCSR(self, resource): Logging.logDebug('Deleting local CSR: %s' % resource.ri) return CSE.dispatcher.deleteResource(resource)
def main(args): Logging.info("HyperParams:") for k, v in args.items(): Logging.info("\t%s: %s" % (k, v)) # Data prep. # X, Y = getData("data/DataFor640/dataset1/", "LinearX.csv", "LinearY.csv") # X = normalize(X) # X, Y = getData("data/DataFor640/dataset1/", "NonLinearX.csv", "NonLinearY.csv") X, Y = getData("data/DataFor640/dataset2/", "Digit_x.csv", "Digit_y.csv") X = normalize(X) train_ind, val_ind = splitData(X, Y, args["k_fold"], 5) X_train, Y_train = X[train_ind], Y[train_ind] X_val, Y_val = X[val_ind], Y[val_ind] Logging.info("X train shape: {}".format(X_train.shape)) Logging.info("Y train shape: {}".format(Y_train.shape)) Logging.info("X val shape: {}".format(X_val.shape)) Logging.info("Y val shape: {}".format(Y_val.shape)) model = train(X_train, Y_train, args) if Y_train.shape[1] == 2: plotDecisionBoundary(model, X_train, Y_train) y_predict = model.predict(X_val) metrics = getPerformanceScores(Y_val, y_predict) Logging.info("Confusion metric: \n{}".format(metrics["CM"])) Logging.info("Accuracy: {:.4f}".format(metrics["accuracy"])) Logging.info("Precision: {}".format( ["%.3f" % f for f in metrics["precision"]])) Logging.info("Recall: {}".format(["%.3f" % f for f in metrics["recall"]])) Logging.info("F1: {}".format(["%.3f" % f for f in metrics["f1"]])) if Y_val.shape[ 1] == 2: # plot ROC curve only for binary classification dataset get_plot_ROC_2(model, X_val, Y_val) test_cost = model.getCost(Y_val, model.forward(X_val)) Logging.info("Test Loss: %.3f" % test_cost) plt.show() return metrics
def shutdown(self): self.stop() Logging.log('RemoteCSEManager shut down')
def importResources(self, path: str = None) -> bool: # Only when the DB is empty else don't imports if CSE.dispatcher.countResources() > 0: Logging.log('Resources already imported, skipping importing') # But we still need the CSI etc of the CSE rss = CSE.dispatcher.retrieveResourcesByType(C.tCSEBase) if rss is not None: Configuration.set('cse.csi', rss[0]['csi']) Configuration.set('cse.ri', rss[0]['ri']) Configuration.set('cse.rn', rss[0]['rn']) return True Logging.logErr('CSE not found') return False # get the originator for the creator attribute of imported resources originator = Configuration.get('cse.originator') # Import if path is None: if Configuration.has('cse.resourcesPath'): path = Configuration.get('cse.resourcesPath') else: Logging.logErr('cse.resourcesPath not set') raise RuntimeError('cse.resourcesPath not set') if not os.path.exists(path): Logging.logWarn('Import directory does not exist: %s' % path) return False Logging.log('Importing resources from directory: %s' % path) self._prepareImporting() # first import the priority resources, like CSE, Admin ACP, Default ACP hasCSE = False hasACP = False for rn in self._firstImporters: fn = path + '/' + rn if os.path.exists(fn): Logging.log('Importing resource: %s ' % fn) jsn = self.readJSONFromFile(fn) r, _ = resourceFromJSON(jsn, create=True, isImported=True) # Check resource creation if not CSE.registration.checkResourceCreation(r, originator): continue CSE.dispatcher.createResource(r) ty = r.ty if ty == C.tCSEBase: Configuration.set('cse.csi', r.csi) Configuration.set('cse.ri', r.ri) Configuration.set('cse.rn', r.rn) hasCSE = True elif ty == C.tACP: hasACP = True # Check presence of CSE and at least one ACP if not (hasCSE and hasACP): Logging.logErr('CSE and/or default ACP missing during import') self._finishImporting() return False # then get the filenames of all other files and sort them. Process them in order filenames = sorted(os.listdir(path)) for fn in filenames: if fn not in self._firstImporters: Logging.log('Importing resource from file: %s' % fn) filename = path + '/' + fn # update an existing resource if 'update' in fn: jsn = self.readJSONFromFile(filename) keys = list(jsn.keys()) if len(keys) == 1 and (k := keys[0]) and 'ri' in jsn[k] and ( ri := jsn[k]['ri']) is not None: r, _, _ = CSE.dispatcher.retrieveResource(ri) if r is not None: CSE.dispatcher.updateResource(r, jsn) # TODO handle error # create a new cresource else: jsn = self.readJSONFromFile(filename) r, _ = resourceFromJSON(jsn, create=True, isImported=True) # Try to get parent resource if r is not None: parent = None if (pi := r.pi) is not None: parent, _, _ = CSE.dispatcher.retrieveResource(pi) # Check resource creation if not CSE.registration.checkResourceCreation( r, originator): continue # Add the resource CSE.dispatcher.createResource(r, parent) else: Logging.logWarn('Unknown resource in file: %s' % fn)
def start(self): if not Configuration.get('cse.enableRemoteCSE'): return; Logging.log('Starting remote CSE connection monitor') self.worker = BackgroundWorker.BackgroundWorker(self.checkInterval, self.connectionMonitorWorker) self.worker.start()
def _callThread(self, *args: Any, **kwargs: Any) -> None: for function in self: try: function(*args, **kwargs) except Exception as e: Logging.logErr(f'{function} - {traceback.format_exc()}')
def shutdown(self) -> bool: global _running _running = False Logging.log('EventManager shut down') return True