def post(url, endpoint, payload):
"""
Make a POST request to the desired URL with the supplied Payload
:param url: URL to make the POST Request to
:type url: str
:param endpoint: Path of the URL to make the request to
:type endpoint: str
:param payload: POST Payload
:type payload: dict
:return: Result JSON, If the request was made successfully
:rtype: str, bool
"""
url_endpoint = url + endpoint
Logging.get_logger(__name__).info('Making POST Request to ' + url_endpoint)
desk_config = Config.load_params()['Desk.com']
if desk_config['auth_method'] == 'oauth':
oauth_session = OAuth1Session(desk_config['client_key'],
client_secret=desk_config['client_secret'],
resource_owner_key=desk_config['resource_owner_key'],
resource_owner_secret=desk_config['resource_owner_secret'])
request = oauth_session.post(url_endpoint, payload)
else:
request = requests.post(url_endpoint, json=payload, auth=(desk_config['username'], desk_config['password']))
try:
return request.json(), request.status_code == 200
except Exception, e:
Logging.get_logger(__name__).error('Problem Getting JSON from POST Request - %s' % e.message)
return []
def get(url, endpoint):
"""
Make a GET request to the desk.com API using the pre-configured authentication method
:param url: URL to make the GET Request to
:type url: str
:param endpoint: Path of the URL to make the request to
:type endpoint: str
:return: Returned JSON
:rtype: str
"""
url_endpoint = url + endpoint
Logging.get_logger(__name__).info('Making GET Request to ' + url_endpoint)
desk_config = Config.load_params()['Desk.com']
if desk_config['auth_method'] == 'oauth':
oauth_session = OAuth1Session(desk_config['client_key'],
client_secret=desk_config['client_secret'],
resource_owner_key=desk_config['resource_owner_key'],
resource_owner_secret=desk_config['resource_owner_secret'])
request = oauth_session.get(url_endpoint)
else:
request = requests.get(url_endpoint, auth=(desk_config['username'], desk_config['password']))
try:
return request.json(), request.status_code == 200
except Exception, e:
Logging.get_logger(__name__).error('Problem Getting JSON from GET Request - %s' % e.message)
return []
def create(self):
conn = libvirt.open(DC.get("virsh", "connect"))
try:
storagePool = conn.storagePoolLookupByName(self.pool)
storagePool.refresh(0)
except:
Logging.errorExit("There is no '%s' libvirt storage pool." % self.pool)
path = os.path.basename(self.path)
size = Units(self.size).convertTo("B")
diskXML = """
<volume>
<name>%s</name>
<capacity>%i</capacity>
<allocation>0</allocation>
<target>
<format type='raw'/>
</target>
</volume>
"""
diskXML = diskXML % (path, size)
try:
storage = storagePool.createXML(diskXML, 0)
except Exception as e:
Logging.errorExit("%s (name: %s)" % (str(e), path))
return storage
class GunControl(object):
def __init__(self, cont):
self.log = Logging("GunControl", "Debug")
self.cont = cont
self.own = cont.owner
self.scene = bge.logic.getCurrentScene()
self.bullets = self.own["Bullets"] if self.own["Bullets"] else 0
self.magazines = self.own["Magazines"] if self.own["Magazines"] else 0
self.reload = self.cont.sensors["Reload"]
self.shoot = self.cont.sensors["Shoot"]
self.aim = self.cont.sensors["Ray"]
self.triggerPull = self.cont.sensors["Trigger"]
self.gun = self.own.parent
self._updateHUD()
self.log.msg("Init Completed.")
def update(self):
if self.aim.positive:
# Add the laser targeting pointer to the scene
bge.render.drawLine(self.own.worldPosition, self.aim.hitPosition, [255, 0, 0])
if self.reload.positive and self.magazines > 0:
self._reload()
self.gun.state = 1 # flip the gun back to shoot mode
if self.bullets > 0:
if self.shoot.positive or self.triggerPull.positive:
self._shoot()
else:
self.gun.state = 2 # gun is empty
self._updateHUD()
def _shoot(self):
self.bullets += -1
if self.aim.positive:
bulletForce = self.scene.addObject("BulletForce", self.own, 3)
bulletForce.worldPosition = Vector(self.aim.hitPosition) + (Vector(self.aim.hitNormal) * 0.01)
bulletHole = self.scene.addObject("BulletHole", self.own, 200)
# position the bullet based on the ray, give a margin factor due to rendering collision
bulletHole.worldPosition = Vector(self.aim.hitPosition) + (Vector(self.aim.hitNormal) * 0.01)
bulletHole.alignAxisToVect(self.aim.hitNormal, 2)
self._activate(self.cont.actuators["Fire"])
self._activate(self.cont.actuators["MuzzleFlash"])
def _reload(self):
# self._activate(self.cont.actuators["Reload"])
self.magazines += -1
self.bullets = self.own["Bullets"] if self.own["Bullets"] else 0
def _updateHUD(self):
bge.logic.sendMessage("Ammo", str(self.bullets))
bge.logic.sendMessage("Clips", str(self.magazines))
if self.magazines == 0 and self.bullets == 0:
bge.logic.sendMessage("GameOver", "GameOver")
def _activate(self, actuator):
self.cont.activate(actuator)
class Parsers:
"""
parse json file, store important stuff
get no of pages-> pages
page_counter <- 1
while(page_counter < pages):
get_page(page_counter)
for recording in recordings:
get_recording_details()
get_audio_file()
page_counter++
"""
MAX_IMAGES_URL = 4
MAX_NO_THREADS = 100
CONFIG_FILE = 'bsrs.cfg'
BIRD_SOUNDS_DIR = 'BirdSounds/'
def __init__(self):
self.fetcher = Fetcher()
self.logger = Logging()
self.config_file = Parsers.CONFIG_FILE
self.config = ConfigParser()
self.load_config_file()
creds = self.get_db_creds()
self.database = MySQLDatabases(hostname=creds['hostname'], username=creds['username'],
password=creds['passwd'], database=creds['db_name'])
#queues
self.birdID_queue = self.wavFile_queue = Queue()
self.soundtype_queue = self.soundURL_queue = Queue()
def get_db_creds(self):
"""
load db creds from config file
"""
hostname = self.config.get('database', 'db_hostname')
username = self.config.get('database', 'db_username')
passwd = self.config.get('database', 'db_password')
db_name = self.config.get('database', 'db_dbname')
return {'hostname': hostname, 'username': username,
'passwd': passwd, 'db_name': db_name}
def load_config_file(self):
"""
Load config file
"""
try:
self.config.read(self.config_file)
info_msg = "Loaded config file %s" % self.config_file
self.logger.write_log('fetcher', 'i', info_msg)
except Exception, e:
info_msg = "config file %s missing" % self.config_file
self.logger.write_log('fetcher', 'e', info_msg)
raise Exception(info_msg)
def create(self):
Logging.info("Creating the network ...")
conn = libvirt.open(DC.get("virsh", "connect"))
configFile = file(self.config, "r")
#print configFile.read()
self.network = conn.networkDefineXML(configFile.read())
self.network.setAutostart(1)
self.network.create()
return
def get_filters(self):
"""
Get all the filters in Desk.com
:return: All Filters in Desk, If the request was completed successfully.
:rtype: str, bool
"""
results, ok = self.get(self.url, 'filters')
Logging.get_logger(__name__).debug('Get Filters returned %s' % str(results))
return results, ok
def check(self):
network = self.parseName()
Logging.info("Checking network '%s'..." % network)
conn = libvirt.open(DC.get("virsh", "connect"))
networks = conn.listNetworks()
if not network in networks:
return True
return False
class GunUdpHandler(object):
def __init__(self):
self.protocolFormat = '<ffffff'
self.log = Logging("GunUDPHandler","Debug")
self.log.msg("Init Completed.")
def parse(self,rawData):
try:
data = struct.unpack(self.protocolFormat,rawData)
#self.log.msg("data: "+str(data[0])+" "+str(data[1]))
return str(data[0])+','+str(data[1])+','+str(data[2])+','+str(data[3])+','+str(data[4])+','+str(data[5])
except Exception as e:
self.log.msg("error: "+str(e))
def resolve_case(self, case_id):
"""
Mark a Case as Resolved
:param case_id: Case to Resolve
:type case_id: int
:return: The Updated Case, If the request was made successfully.
:rtype: str, bool
"""
Logging.get_logger(__name__).info("Resolving Case with ID: %d" % case_id)
result, ok = self.patch(self.url, 'cases/' + str(case_id), {"status": "resolved"})
return result, ok
def run(self):
if self.configPath and self.config.sections():
self.start()
self.login()
self.setNetwork()
output = self.getNetwork()
self.stop()
return output
else:
Logging.warning("Network settings not set. Skipping setting"
" network on guest '%s'.", self.guest.getHostName())
return {}
def get_groups(self):
"""
Get all the Groups in Desk.com
:return: All Groups in Desk.com, If the request was successful
:rtype: str, bool
"""
results, ok = self.get(self.url, 'groups')
if ok:
results = results['_embedded']['entries']
Logging.get_logger(__name__).debug('Get Groups returned %s' % str(results))
return results, ok
def __init__(self, justPlots = False):
self.__name__ = "Core"
self.configManager = ConfigurationManager()
# These return True of False depending on whether loading the conf was a success.
# It should be checked if the conf was loaded successfully and failures should be logged.
self.configManager.loadConf(CONFIG_CORE, True)
self.configManager.loadConf(CONFIG_SETTINGS, True)
self.configManager.loadConf(CONFIG_FORMS, True)
self.configManager.loadConf(CONFIG_URLMAP, True)
self.configManager.loadConf(CONFIG_MESSAGES, True)
self.moduleManager = ModuleManager(self)
self.settingsManager = SettingsManager(self)
self.clientManager = ClientManager(self)
self.sensorManager = SensorManager(self)
self.deviceManager = DeviceManager(self)
self.taskManager = TaskManager(self)
self.messageManager = MessageManager(self)
self.logging = Logging(self)
if self.settingsManager.equals("plottype", "matplotlib"):
from Plot import Plot
self.plot = Plot(self)
self.protocol = Protocol(self)
if not justPlots: self.connection = Connection(self)
if not justPlots: self.scheduler = Scheduler()
if not justPlots: self.webServer = WebServer(self.connection.getLocalIP(), self.settingsManager.getValueByName("listenport")) # Currently binds to localhost. But this needs to be fixed so other connections can be listened to too.
def umount(self):
if os.path.exists(self.mountPoint) and os.path.isdir(self.mountPoint):
Logging.debug("Unmounting %s." % self.mountPoint)
mountProcess = subprocess.Popen(["/bin/umount", self.mountPoint])
mountProcess.wait()
if mountProcess.returncode == 0 and len(os.listdir(self.mountPoint)) == 0:
if self.createdMountPoint:
Logging.debug("Deleting mounting point: %s" % self.mountPoint)
os.rmdir(self.mountPoint)
self.createdMountPoint = False
return True
return False
def add_note_to_case(self, case_id, note):
"""
Add a note to the Supplied Case
:param case_id: Case to add note to
:type case_id: int
:param note: Note to be added to Case
:type note: str
:return: The Updated Case, If the request was made successfully.
:rtype: str, bool
"""
Logging.get_logger(__name__).info("Adding Note to Case with ID: %d" % case_id)
result, ok = self.post(self.url, 'cases/' + str(case_id) + '/notes',
{"body": note})
return result, ok
class GameOver(object):
'''
this class is the main entry point and control of the game scene
'''
def __init__(self,cont):
self.cont = cont
self.own = cont.owner
self.scene = bge.logic.getCurrentScene()
self.log = Logging("GameOver","Debug")
self.showScoreboard = self.cont.actuators['GoScoreboard']
self.showScoreboard.scene="Scoreboard"
self.kbd = self.cont.sensors['Keyboard']
self.log.msg('init completed')
def update(self):
if self.kbd.positive:
self.cont.activate(self.showScoreboard)
def __init__(self):
self.host = "0.0.0.0"
self.port = 10001
self.protocol = ProtocolHandler()
self.socketClient = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.socketClient.bind((self.host, self.port))
self.socketClient.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 256)
self.socketClient.setblocking(0)
self.log = Logging("Networking", "Debug")
self.log.msg("Init Completed.")
def download(self):
fileName = self.path.rsplit("/", 1)
filePath = "%s/%i-%s" % (self.tempStorage, hash(self), fileName[1])
if os.path.exists(filePath):
self.path = filePath
self.downloaded = True
return self.downloaded
Logging.debug("Trying download %s to %s." % (self.path, filePath))
try:
httpRequest = urllib2.urlopen(self.path)
except urllib2.HTTPError as e:
return None
if (not os.path.exists(self.tempStorage)
or not os.access(self.tempStorage, os.W_OK)
or System.getFreeSpace(self.tempStorage) < int(httpRequest.info().get("Content-Length"))):
return None
try:
iso = file(filePath, "w")
while 1:
buf = httpRequest.read(16*1024)
if not buf:
break
iso.write(buf)
iso.close()
except IOError as e:
return None
self.path = filePath
self.downloaded = os.path.exists(filePath)
return self.downloaded
def __init__(self,cont):
self.cont = cont
self.own = cont.owner
self.scene = bge.logic.getCurrentScene()
self.log = Logging("GameOver","Debug")
self.showScoreboard = self.cont.actuators['GoScoreboard']
self.showScoreboard.scene="Scoreboard"
self.kbd = self.cont.sensors['Keyboard']
self.log.msg('init completed')
def __init__(self,cont):
self.cont = cont
self.own = cont.owner
self.log = Logging("Splash","Debug")
self.scene = bge.logic.getCurrentScene()
self.title = self.scene.objects["Title"]
self.title.text = "SAND-Matrix"
self.log.msg("Init complete.")
def __init__(self,cont):
self.cont = cont
self.own = cont.owner
self.scene = bge.logic.getCurrentScene()
self.log = Logging("Menu","Debug")
self.startGame = self.cont.actuators['StartGame']
self.startGame.scene="Main"
self.mouse = self.cont.sensors['Mouse']
self.trigger = self.cont.sensors['Trigger']
self.log.msg('init completed')
class Splash(object):
'''
this class is the Splash
'''
def __init__(self,cont):
self.cont = cont
self.own = cont.owner
self.log = Logging("Splash","Debug")
self.scene = bge.logic.getCurrentScene()
self.title = self.scene.objects["Title"]
self.title.text = "SAND-Matrix"
self.log.msg("Init complete.")
def update(self):
self._handleMessage()
def _handleMessage(self):
pass
class Networking(object):
def __init__(self):
self.host = "0.0.0.0"
self.port = 10001
self.protocol = ProtocolHandler()
self.socketClient = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.socketClient.bind((self.host, self.port))
self.socketClient.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 256)
self.socketClient.setblocking(0)
self.log = Logging("Networking", "Debug")
self.log.msg("Init Completed.")
def update(self):
try:
rawData, SRIP = self.socketClient.recvfrom(256)
data = self.protocol.parse(rawData)
self.log.msg(data)
bge.logic.sendMessage("SpawnEnemy", data, "Source", "Source")
except:
pass
class Menu(object):
'''
this class is the main entry point and control of the game scene
'''
def __init__(self,cont):
self.cont = cont
self.own = cont.owner
self.scene = bge.logic.getCurrentScene()
self.log = Logging("Menu","Debug")
self.startGame = self.cont.actuators['StartGame']
self.startGame.scene="Main"
self.mouse = self.cont.sensors['Mouse']
self.trigger = self.cont.sensors['Trigger']
self.log.msg('init completed')
def update(self):
if self.mouse.positive or self.trigger.positive:
self.log.msg('here')
self.cont.activate(self.startGame)
def get_cases(self, case_filter=''):
"""
Get all cases that match the supplied filter. If no filter is supplied, all cases will be fetched.
:param case_filter: Either Customer ID, Company ID, or Filter ID
:type case_filter: str
:return: All Cases that match the filter, If the request was completed successfully.
:rtype: str, bool
"""
results, ok = self.get(self.url, 'cases' + case_filter)
all_cases = results['_embedded']['entries']
Logging.get_logger(__name__).debug('Get Cases returned %s' % str(results))
while results['_links']['next'] is not None:
results, ok = self.get(self.url, results['_links']['next']['href'].split('/')[-1])
Logging.get_logger(__name__).debug('Get Cases returned %s' % str(results))
all_cases.append(results['_embedded']['entries'])
Logging.get_logger(__name__).info('Fetched a total of %d cases.' % len(all_cases))
marshaled_results = Marshal.desk_case(all_cases)
for case in marshaled_results:
case.customer = self.get_customer(case.customer_id)[0]
return marshaled_results, ok
def install(self, handler=DefaultHandler):
self.setValues()
destroy = None
servers = self.startHttpd()
master, slave = pty.openpty()
stdin = os.fdopen(master, "w")
logPath = self.log % self.values
Logging.debug("Trying write install log of VM '%s' to %s."
% (self.guest.getHostName(), logPath))
stdout = open(logPath, "w")
try:
self.cmdInstall = self.cmdInstall.replace("\n", " ")
Logging.info("Trying install guest '%s'..." % self.guest.getHostName())
Logging.debug(self.cmdInstall % self.values)
process = subprocess.Popen(self.cmdInstall % self.values, shell=True,
stdin=slave, stdout=stdout, stderr=stdout,
close_fds=True)
analyzator = handler(stdin, stdout, process)
analyzator.handle()
except InstallationError as e:
destroy = subprocess.Popen(self.cmdDestroy % self.values, shell=True)
Logging.info("Check installation log for details: %s" % logPath)
raise e
finally:
if servers[0]: servers[0].kill()
if servers[1]: servers[1].kill()
if destroy:
return not destroy.wait()
Logging.info("Guest '%s' installed." % self.guest.getHostName())
return
def insertDNS(self, dnsDict):
conn = libvirt.open(DC.get("virsh", "connect"))
xmlDesc = self.network.XMLDesc(0)
Logging.debug("Inserting DNS entries: %s" % str(dnsDict))
localDns = self.makeDNS(dnsDict)
networkXml = xml.parseString(xmlDesc)
dns = networkXml.getElementsByTagName("dns")
if dns:
dns = dns[0]
for entry in localDns.getElementsByTagName("host"):
dns.appendChild(entry)
else:
networkXml.documentElement.appendChild(localDns)
self.network.destroy()
self.network.undefine()
self.network = conn.networkDefineXML(networkXml.documentElement.toxml())
self.network.setAutostart(1)
self.network.create()
values = {'addr': self.getIPv6Address(), 'prefix': self.getIPv6Prefix(),
'interface': self.getInterface()}
if (DC.has_option("network", "manual-route")
and DC.get("network", "manual-route", True)):
cmd = DC.get("network", "manual-route", False, values)
cmd = cmd.replace("\n", " ")
subprocess.Popen(cmd, shell=True)
else:
Logging.warning("Manual route not set. Skipping.")
return
class GunUdpListener(object):
def __init__(self):
self.host = "0.0.0.0"
self.port = 10002
self.protocol = GunUdpHandler()
self.socketClient = socket.socket(socket.AF_INET,socket.SOCK_DGRAM)
self.socketClient.bind((self.host, self.port))
self.socketClient.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 256)
self.socketClient.setblocking(0)
self.log = Logging("GunUdpListener","Debug")
self.log.msg("Init Completed.")
def update(self):
try:
rawData, SRIP = self.socketClient.recvfrom(256)
#self.log.msg(rawData)
data = self.protocol.parse(rawData)
#self.log.msg(data)
bge.logic.sendMessage("GunPos", data)
except Exception as e:
#self.log.msg(e)
pass
def __init__(self):
self.fetcher = Fetcher()
self.logger = Logging()
self.config_file = Parsers.CONFIG_FILE
self.config = ConfigParser()
self.load_config_file()
creds = self.get_db_creds()
self.database = MySQLDatabases(hostname=creds['hostname'], username=creds['username'],
password=creds['passwd'], database=creds['db_name'])
#queues
self.birdID_queue = self.wavFile_queue = Queue()
self.soundtype_queue = self.soundURL_queue = Queue()
class ProtocolHandler(object):
"""
Struct-based Binary Protocol Definition (UDP)
=============================================
< = little endian
L = Source IP (integer)
L = Destination IP (integer)
"""
def __init__(self):
self.protocolFormat = "<LL"
self.log = Logging("ProtocolHandler", "Debug")
self.log.msg("Init Completed.")
def parse(self, rawData):
data = struct.unpack(self.protocolFormat, rawData)
return str(data[0]) + "," + str(data[1])
def _dec2dot(numbericIP):
if type(numbericIP) == types.StringType and not numbericIP.isdigit():
return None
numIP = long(numbericIP)
return "%d.%d.%d.%d" % ((numIP >> 24) & 0xFF, (numIP >> 16) & 0xFF, (numIP >> 8) & 0xFF, numIP & 0xFF)
def __init__(self) -> None:
Logging.log('RegistrationManager initialized')
lr=conf.learning_rate,
weight_decay=conf.weight_decay)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=5,
gamma=0.8)
############################## PREPARE DATASET ##############################
print('System start to load data...')
t0 = time()
train_data, val_data, test_data = data_utils.load_all()
t1 = time()
print('Data has been loaded successfully, cost:%.4fs' % (t1 - t0))
########################### FIRST TRAINING #####################################
check_dir('%s/train_%s_mrg_id_x.log' % (conf.out_path, conf.data_name))
log = Logging('%s/train_%s_mrg_id_06.py' % (conf.out_path, conf.data_name))
train_model_path = '%s/train_%s_mrg_id_06.mod' % (conf.out_path,
conf.data_name)
# prepare data for the training stage
train_dataset = data_utils.TrainData(train_data)
val_dataset = data_utils.TrainData(val_data)
test_dataset = data_utils.TrainData(test_data)
train_batch_sampler = data.BatchSampler(data.RandomSampler(
range(train_dataset.length)),
batch_size=conf.batch_size,
drop_last=False)
val_batch_sampler = data.BatchSampler(data.RandomSampler(
range(val_dataset.length)),
batch_size=conf.batch_size,
def __init__(self) -> None:
Logging.log('Importer initialized')
def handleDeleteRequest(self, request: CSERequest, id: str,
originator: str) -> Result:
""" Handle a DELETE request. Delete the latest resource. """
Logging.logDebug('Deleting latest CIN from CNT')
if (r := self._getLatest()) is None:
return Result(rsc=RC.notFound, dbg='no instance for <latest>')
def shutdown(self) -> bool:
Logging.log('GroupManager shut down')
return True
model.cuda()
#optimizer = torch.optim.SGD(model.parameters(), lr=conf.learning_rate, weight_decay=conf.weight_decay)
optimizer = torch.optim.Adam(model.parameters(),
lr=conf.learning_rate,
weight_decay=conf.weight_decay)
############################## PREPARE DATASET ##############################
print('System start to load data...')
t0 = time()
train_data, val_data, test_data = data_utils.load_all()
t1 = time()
print('Data has been loaded successfully, cost:%.4fs' % (t1 - t0))
########################### FIRST TRAINING #####################################
check_dir('%s/train_%s_pmf_id_x.log' % (conf.out_path, conf.data_name))
log = Logging('%s/train_%s_ncf_id_X1.log' %
(conf.out_path, conf.data_name))
train_model_path = '%s/train_%s_ncf_id_X1.mod' % (conf.out_path,
conf.data_name)
# prepare data for the training stage
train_dataset = data_utils.TrainData(train_data)
val_dataset = data_utils.TrainData(val_data)
test_dataset = data_utils.TrainData(test_data)
train_batch_sampler = data.BatchSampler(data.RandomSampler(
range(train_dataset.length)),
batch_size=conf.batch_size,
drop_last=False)
val_batch_sampler = data.BatchSampler(data.RandomSampler(
range(val_dataset.length)),
batch_size=conf.batch_size,
scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=5,
gamma=0.8)
############################## PREPARE DATASET ##############################
print('System start to load data...')
t0 = time()
train_data, val_data, test_data, \
train_user_historical_review_dict, train_item_historical_review_dict = data_utils.load_all()
t1 = time()
print('Data has been loaded successfully, cost:%.4fs' % (t1 - t0))
########################### FIRST TRAINING #####################################
check_dir('%s/train_%s_aspect_rating_1_id_x.log' %
(conf.out_path, conf.data_name))
log = Logging('%s/train_%s_aspect_rating_1_id_65.py' %
(conf.out_path, conf.data_name))
train_model_path = '%s/train_%s_aspect_rating_1_id_65.mod' % (
conf.out_path, conf.data_name)
# prepare data for the training stage
train_dataset = data_utils.TrainData(train_data,
train_user_historical_review_dict,
train_item_historical_review_dict,
train_data)
val_dataset = data_utils.TrainData(val_data,
train_user_historical_review_dict,
train_item_historical_review_dict,
train_data)
test_dataset = data_utils.TrainData(test_data,
train_user_historical_review_dict,
train_item_historical_review_dict,
def _runNonBlockingRequestSync(self, request: CSERequest,
reqRi: str) -> bool:
""" Execute the actual request and store the result in the respective <request> resource.
"""
Logging.logDebug('Executing nonBlockingRequestSync')
return self._executeOperation(request, reqRi).status
def __init__(self) -> None:
self.enableTransit = Configuration.get('cse.enableTransitRequests')
self.flexBlockingBlocking = Configuration.get(
'cse.flexBlockingPreference') == 'blocking'
Logging.log('RequestManager initialized')
from fm import fm
model = fm()
#model.load_state_dict(torch.load('/content/drive/My Drive/task/aspect_based_rs/out/model/train_amazon_clothing_fm_id_2.mod'))
model.cuda()
optimizer = torch.optim.Adam(model.parameters(),
lr=conf.lr,
weight_decay=conf.weight_decay)
scheduler = torch.optim.lr_scheduler.StepLR(optimizer,
step_size=5,
gamma=0.8)
########################### FIRST TRAINING #####################################
check_dir('%s/train_%s_fm_id_x.log' % (conf.out_path, conf.data_name))
log = Logging('%s/train_%s_fm_id_01.log' % (conf.out_path, conf.data_name))
train_model_path = '%s/train_%s_fm_id_01.mod' % (conf.out_path,
conf.data_name)
# prepare data for the training stage
train_dataset = data_utils.TrainData(train_data)
val_dataset = data_utils.TrainData(val_data)
test_dataset = data_utils.TrainData(test_data)
train_batch_sampler = data.BatchSampler(data.RandomSampler(
range(train_dataset.length)),
batch_size=conf.batch_size,
drop_last=False)
val_batch_sampler = data.BatchSampler(data.RandomSampler(
range(val_dataset.length)),
batch_size=conf.batch_size,
reqRi: str) -> bool:
""" Execute the actual request and store the result in the respective <request> resource.
"""
Logging.logDebug('Executing nonBlockingRequestSync')
return self._executeOperation(request, reqRi).status
def _runNonBlockingRequestAsync(self, request: CSERequest,
reqRi: str) -> bool:
""" Execute the actual request and store the result in the respective <request> resource.
In addition notify the notification targets.
"""
Logging.logDebug('Executing nonBlockingRequestAsync')
if not (result := self._executeOperation(request, reqRi)).status:
return False
Logging.logDebug(
'Sending result notifications for nonBlockingRequestAsynch')
# TODO move the notification to the notificationManager
# The result contains the request resource (the one from the actual operation).
# So we can just copy the individual attributes
originator = result.resource['ors/fr']
responseNotification = {
'm2m:rsp': {
'rsc': result.resource['ors/rsc'],
'rqi': result.resource['ors/rqi'],
'pc': result.resource['ors/pc'],
'to': result.resource['ors/to'],
'fr': originator,
'rvi': request.headers.releaseVersionIndicator
}
}
class FCNT(AnnounceableResource):
def __init__(self,
dct: JSON = None,
pi: str = None,
fcntType: str = None,
create: bool = False) -> None:
super().__init__(T.FCNT,
dct,
pi,
tpe=fcntType,
create=create,
attributePolicies=attributePolicies)
self.resourceAttributePolicies = fcntPolicies # only the resource type's own policies
if self.dict is not None:
self.setAttribute('cs', 0, overwrite=False)
# "current" attributes are added when necessary in the validate() method
# Indicates whether this FC has flexContainerInstances.
# Might change during the lifetime of a resource. Used for optimization
self.hasInstances = False
self.__validating = False
self.ignoreAttributes = self.internalAttributes + [
'acpi', 'cbs', 'cni', 'cnd', 'cs', 'cr', 'ct', 'et', 'lt', 'mbs',
'mia', 'mni', 'or', 'pi', 'ri', 'rn', 'st', 'ty', 'at'
]
# Enable check for allowed sub-resources
def canHaveChild(self, resource: Resource) -> bool:
return super()._canHaveChild(
resource,
[
T.CNT, T.FCNT, T.SUB
# FlexContainerInstances are added by the flexContainer itself
])
def activate(self, parentResource: Resource, originator: str) -> Result:
if not (res := super().activate(parentResource, originator)).status:
return res
# register latest and oldest virtual resources
Logging.logDebug(
f'Registering latest and oldest virtual resources for: {self.ri}')
if self.hasInstances:
# add latest
resource = Factory.resourceFromDict(
{}, pi=self.ri,
ty=T.FCNT_LA).resource # rn is assigned by resource itself
if (res :=
CSE.dispatcher.createResource(resource)).resource is None:
return Result(status=False, rsc=res.rsc, dbg=res.dbg)
# add oldest
resource = Factory.resourceFromDict(
{}, pi=self.ri,
ty=T.FCNT_OL).resource # rn is assigned by resource itself
if (res :=
CSE.dispatcher.createResource(resource)).resource is None:
return Result(status=False, rsc=res.rsc, dbg=res.dbg)
def _removeACP(self, srn: str, resource: Resource) -> Tuple[Resource, int, str]:
""" Remove an ACP created during registration before. """
if (acpRes := CSE.dispatcher.retrieveResource(id=srn))[1] != C.rcOK:
Logging.logWarn('Could not find ACP: %s' % srn) # ACP not found, either not created or already deleted
def shutdown(self) -> None:
Logging.log('RegistrationManager shut down')
def start(conf, data, model, evaluate):
log_dir = os.path.join(os.getcwd(), 'log')
if not os.path.exists(log_dir):
os.makedirs(log_dir)
# define log name
log_path = os.path.join(
os.getcwd(), 'log/%s_%s.log' % (conf.data_name, conf.model_name))
# start to prepare data for training and evaluating
data.initializeRankingHandle()
d_train, d_val, d_test, d_test_eva = data.train, data.val, data.test, data.test_eva
print('System start to load data...')
t0 = time()
d_train.initializeRankingTrain()
d_val.initializeRankingVT()
d_test.initializeRankingVT()
d_test_eva.initalizeRankingEva()
t1 = time()
print('Data has been loaded successfully, cost:%.4fs' % (t1 - t0))
# prepare model necessary data.
data_dict = d_train.prepareModelSupplement(model)
model.inputSupply(data_dict)
model.startConstructGraph()
# standard tensorflow running environment initialize
tf_conf = tf.ConfigProto()
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
tf_conf.gpu_options.allow_growth = True
sess = tf.Session(config=tf_conf)
sess.run(model.init)
if conf.pretrain_flag == 1:
model.saver.restore(sess, conf.pre_model)
# set debug_flag=0, doesn't print any results
log = Logging(log_path)
print()
log.record('Following will output the evaluation of the model:')
# Start Training !!!
for epoch in range(1, conf.epochs + 1):
# optimize model with training data and compute train loss
tmp_train_loss = []
t0 = time()
#tmp_total_list = []
while d_train.terminal_flag:
d_train.getTrainRankingBatch()
d_train.linkedMap()
train_feed_dict = {}
for (key, value) in model.map_dict['train'].items():
train_feed_dict[key] = d_train.data_dict[value]
[sub_train_loss, _] = sess.run(\
[model.map_dict['out']['train'], model.opt], feed_dict=train_feed_dict)
tmp_train_loss.append(sub_train_loss)
train_loss = np.mean(tmp_train_loss)
t1 = time()
# compute val loss and test loss
d_val.getVTRankingOneBatch()
d_val.linkedMap()
val_feed_dict = {}
for (key, value) in model.map_dict['val'].items():
val_feed_dict[key] = d_val.data_dict[value]
val_loss = sess.run(model.map_dict['out']['val'],
feed_dict=val_feed_dict)
d_test.getVTRankingOneBatch()
d_test.linkedMap()
test_feed_dict = {}
for (key, value) in model.map_dict['test'].items():
test_feed_dict[key] = d_test.data_dict[value]
test_loss = sess.run(model.map_dict['out']['test'],
feed_dict=test_feed_dict)
t2 = time()
# start evaluate model performance, hr and ndcg
def getPositivePredictions():
d_test_eva.getEvaPositiveBatch()
d_test_eva.linkedRankingEvaMap()
eva_feed_dict = {}
for (key, value) in model.map_dict['eva'].items():
eva_feed_dict[key] = d_test_eva.data_dict[value]
positive_predictions = sess.run(model.map_dict['out']['eva'],
feed_dict=eva_feed_dict)
return positive_predictions
def getNegativePredictions():
negative_predictions = {}
terminal_flag = 1
while terminal_flag:
batch_user_list, terminal_flag = d_test_eva.getEvaRankingBatch(
)
d_test_eva.linkedRankingEvaMap()
eva_feed_dict = {}
for (key, value) in model.map_dict['eva'].items():
eva_feed_dict[key] = d_test_eva.data_dict[value]
index = 0
tmp_negative_predictions = np.reshape(
sess.run(model.map_dict['out']['eva'],
feed_dict=eva_feed_dict), [-1, conf.num_evaluate])
for u in batch_user_list:
negative_predictions[u] = tmp_negative_predictions[index]
index = index + 1
return negative_predictions
tt2 = time()
index_dict = d_test_eva.eva_index_dict
positive_predictions = getPositivePredictions()
negative_predictions = getNegativePredictions()
d_test_eva.index = 0 # !!!important, prepare for new batch
hr, ndcg = evaluate.evaluateRankingPerformance(\
index_dict, positive_predictions, negative_predictions, conf.topk, conf.num_procs)
tt3 = time()
# print log to console and log_file
log.record('Epoch:%d, compute loss cost:%.4fs, train loss:%.4f, val loss:%.4f, test loss:%.4f' % \
(epoch, (t2-t0), train_loss, val_loss, test_loss))
log.record('Evaluate cost:%.4fs, hr:%.4f, ndcg:%.4f' %
((tt3 - tt2), hr, ndcg))
## reset train data pointer, and generate new negative data
d_train.generateTrainNegative()
def shutdown(self) -> bool:
Logging.log('RequestManager shut down')
return True
review_optimizer = torch.optim.Adam(model.parameters(),
lr=conf.learning_rate)
rating_optimizer = torch.optim.Adam(model.parameters(),
lr=conf.learning_rate,
weight_decay=conf.weight_decay)
############################## PREPARE DATASET ##############################
print('System start to load data...')
t0 = time()
train_data, val_data, test_data = data_utils.load_all()
t1 = time()
print('Data has been loaded successfully, cost:%.4fs' % (t1 - t0))
########################### FIRST TRAINING #####################################
check_dir('%s/train_%s_lm_id_x.py' % (conf.out_path, conf.data_name))
log = Logging('%s/train_%s_mrg_id_X.py' % (conf.out_path, conf.data_name))
train_model_path = '%s/train_%s_mrg_id_X' % (conf.out_path, conf.data_name)
# prepare data for the training stage
train_dataset = data_utils.TrainData(train_data)
train_batch_sampler = data.BatchSampler(data.RandomSampler(\
range(train_dataset.length)), batch_size=conf.batch_size, drop_last=False)
val_dataset = data_utils.TrainData(val_data)
val_batch_sampler = data.BatchSampler(data.SequentialSampler(\
range(val_dataset.length)), batch_size=conf.batch_size, drop_last=False)
test_dataset = data_utils.TrainData(test_data)
test_batch_sampler = data.BatchSampler(data.SequentialSampler(\
range(test_dataset.length)), batch_size=conf.batch_size, drop_last=False)
def __init__(self, message):
if (not isinstance(message, "")) or message == "":
self.message = u"帐号密码错误"
else:
self.message = message
Logging.error(self.message)
def shutdown(self):
Logging.log('GroupManager shut down')
return (CSEBase.CSEBase(jsn), C.rcOK)
#########################################################################
#
# Handling of Transit requests. Forward requests to the resp. remote CSE's.
#
# Forward a Retrieve request to a remote CSE
def handleTransitRetrieveRequest(self, request, id, origin):
if (url := self._getForwardURL(id)) is None:
return (None, C.rcNotFound)
if len(request.args) > 0: # pass on other arguments, for discovery
url += '?' + urllib.parse.urlencode(request.args)
Logging.log('Forwarding Retrieve/Discovery request to: %s' % url)
return CSE.httpServer.sendRetrieveRequest(url, origin)
# Forward a Create request to a remote CSE
def handleTransitCreateRequest(self, request, id, origin, ty):
if (url := self._getForwardURL(id)) is None:
return (None, C.rcNotFound)
Logging.log('Forwarding Create request to: %s' % url)
return CSE.httpServer.sendCreateRequest(url, origin, data=request.data, ty=ty)
# Forward a Update request to a remote CSE
def handleTransitUpdateRequest(self, request, id, origin):
if (url := self._getForwardURL(id)) is None:
return (None, C.rcNotFound)
""" This is the Starting Point of Kijiji Web Scraping Script Environment Setup """
import sys
from datetime import datetime
from Logging import Logging
from SetupEnvironment import SetupEnvironment
""" Getting Basic Logging Options """
logger = Logging().get_logger("setup")
if logger != None:
# Checking Python Version
python_major_version = sys.version_info.major
if python_major_version < 3:
logger.critical(
"Setup Module : Version Issue : Please install Python Version 3 or greater to execute this script"
)
else:
logger.debug("Setup Module : Correct Python Version Found")
logger.debug("Setup Module : Preparing Script Environment")
# Staring Environment Setup
setupEnvironment = SetupEnvironment(logger)
setupEnvironment.installAndUnpgradeLibraries()
logger.debug("Setup Module : Environment Setup Completed")
else:
print("Setup Module : Critical : Logging Setup Could Not Be Completed")
print("Setup Module : Critical : Process will Exit")
print("Setup Module : Critical : Contact Administrator For Resolution")
def _updateLocalCSR(self, localCSR, remoteCSE):
Logging.logDebug('Updating local CSR: %s' % localCSR.rn)
# copy attributes
self._copyCSE2CSE(localCSR, remoteCSE)
return CSE.dispatcher.updateResource(localCSR)
def __init__(self) -> None:
# Add delete event handler because we like to monitor the resources in mid
CSE.event.addHandler(CSE.event.deleteResource,
self.handleDeleteEvent) # type: ignore
Logging.log('GroupManager initialized')
def _deleteLocalCSR(self, resource):
Logging.logDebug('Deleting local CSR: %s' % resource.ri)
return CSE.dispatcher.deleteResource(resource)
def main(args):
Logging.info("HyperParams:")
for k, v in args.items():
Logging.info("\t%s: %s" % (k, v))
# Data prep.
# X, Y = getData("data/DataFor640/dataset1/", "LinearX.csv", "LinearY.csv")
# X = normalize(X)
# X, Y = getData("data/DataFor640/dataset1/", "NonLinearX.csv", "NonLinearY.csv")
X, Y = getData("data/DataFor640/dataset2/", "Digit_x.csv", "Digit_y.csv")
X = normalize(X)
train_ind, val_ind = splitData(X, Y, args["k_fold"], 5)
X_train, Y_train = X[train_ind], Y[train_ind]
X_val, Y_val = X[val_ind], Y[val_ind]
Logging.info("X train shape: {}".format(X_train.shape))
Logging.info("Y train shape: {}".format(Y_train.shape))
Logging.info("X val shape: {}".format(X_val.shape))
Logging.info("Y val shape: {}".format(Y_val.shape))
model = train(X_train, Y_train, args)
if Y_train.shape[1] == 2:
plotDecisionBoundary(model, X_train, Y_train)
y_predict = model.predict(X_val)
metrics = getPerformanceScores(Y_val, y_predict)
Logging.info("Confusion metric: \n{}".format(metrics["CM"]))
Logging.info("Accuracy: {:.4f}".format(metrics["accuracy"]))
Logging.info("Precision: {}".format(
["%.3f" % f for f in metrics["precision"]]))
Logging.info("Recall: {}".format(["%.3f" % f for f in metrics["recall"]]))
Logging.info("F1: {}".format(["%.3f" % f for f in metrics["f1"]]))
if Y_val.shape[
1] == 2: # plot ROC curve only for binary classification dataset
get_plot_ROC_2(model, X_val, Y_val)
test_cost = model.getCost(Y_val, model.forward(X_val))
Logging.info("Test Loss: %.3f" % test_cost)
plt.show()
return metrics
def shutdown(self):
self.stop()
Logging.log('RemoteCSEManager shut down')
def importResources(self, path: str = None) -> bool:
# Only when the DB is empty else don't imports
if CSE.dispatcher.countResources() > 0:
Logging.log('Resources already imported, skipping importing')
# But we still need the CSI etc of the CSE
rss = CSE.dispatcher.retrieveResourcesByType(C.tCSEBase)
if rss is not None:
Configuration.set('cse.csi', rss[0]['csi'])
Configuration.set('cse.ri', rss[0]['ri'])
Configuration.set('cse.rn', rss[0]['rn'])
return True
Logging.logErr('CSE not found')
return False
# get the originator for the creator attribute of imported resources
originator = Configuration.get('cse.originator')
# Import
if path is None:
if Configuration.has('cse.resourcesPath'):
path = Configuration.get('cse.resourcesPath')
else:
Logging.logErr('cse.resourcesPath not set')
raise RuntimeError('cse.resourcesPath not set')
if not os.path.exists(path):
Logging.logWarn('Import directory does not exist: %s' % path)
return False
Logging.log('Importing resources from directory: %s' % path)
self._prepareImporting()
# first import the priority resources, like CSE, Admin ACP, Default ACP
hasCSE = False
hasACP = False
for rn in self._firstImporters:
fn = path + '/' + rn
if os.path.exists(fn):
Logging.log('Importing resource: %s ' % fn)
jsn = self.readJSONFromFile(fn)
r, _ = resourceFromJSON(jsn, create=True, isImported=True)
# Check resource creation
if not CSE.registration.checkResourceCreation(r, originator):
continue
CSE.dispatcher.createResource(r)
ty = r.ty
if ty == C.tCSEBase:
Configuration.set('cse.csi', r.csi)
Configuration.set('cse.ri', r.ri)
Configuration.set('cse.rn', r.rn)
hasCSE = True
elif ty == C.tACP:
hasACP = True
# Check presence of CSE and at least one ACP
if not (hasCSE and hasACP):
Logging.logErr('CSE and/or default ACP missing during import')
self._finishImporting()
return False
# then get the filenames of all other files and sort them. Process them in order
filenames = sorted(os.listdir(path))
for fn in filenames:
if fn not in self._firstImporters:
Logging.log('Importing resource from file: %s' % fn)
filename = path + '/' + fn
# update an existing resource
if 'update' in fn:
jsn = self.readJSONFromFile(filename)
keys = list(jsn.keys())
if len(keys) == 1 and (k :=
keys[0]) and 'ri' in jsn[k] and (
ri := jsn[k]['ri']) is not None:
r, _, _ = CSE.dispatcher.retrieveResource(ri)
if r is not None:
CSE.dispatcher.updateResource(r, jsn)
# TODO handle error
# create a new cresource
else:
jsn = self.readJSONFromFile(filename)
r, _ = resourceFromJSON(jsn, create=True, isImported=True)
# Try to get parent resource
if r is not None:
parent = None
if (pi := r.pi) is not None:
parent, _, _ = CSE.dispatcher.retrieveResource(pi)
# Check resource creation
if not CSE.registration.checkResourceCreation(
r, originator):
continue
# Add the resource
CSE.dispatcher.createResource(r, parent)
else:
Logging.logWarn('Unknown resource in file: %s' % fn)
def start(self):
if not Configuration.get('cse.enableRemoteCSE'):
return;
Logging.log('Starting remote CSE connection monitor')
self.worker = BackgroundWorker.BackgroundWorker(self.checkInterval, self.connectionMonitorWorker)
self.worker.start()
def _callThread(self, *args: Any, **kwargs: Any) -> None:
for function in self:
try:
function(*args, **kwargs)
except Exception as e:
Logging.logErr(f'{function} - {traceback.format_exc()}')
def shutdown(self) -> bool:
global _running
_running = False
Logging.log('EventManager shut down')
return True
