def gameExists(self, uniqueurl): for i in range(0, len(self.sessions)): if self.sessions[i][0] == int(uniqueurl): Logger.log('Game {0} exists'.format(uniqueurl)) return i Logger.warn('Game {0} doesn\'t exist'.format(uniqueurl)) return None
def __main__(): logger = Logger() logger.set_debug_level('info') config= Config() u_base_path = config.base_path hddb = HoerdatDB(config.hoerdat_db_path, {'debug': 0}) dupmerge = DupMerge( config.md5_db_path, u_base_path, {'logger': logger}, ) hd = Hoerdat(hddb) for file, md5 in dupmerge.all_files_md5(): if not hd.set_filename(file, md5): continue if DONT_RESCAN_PRESENT and hd.data['hoerdat'].get_data('hoerdat_id'): continue print file, hd.data['hoerdat'].get_data('hoerdat_id') fetch_result = hd.fetch_data() best_match = hd.get_best_match(fetch_result) if best_match: hd.merge_data(best_match[0], best_match[1]) hd.update() print "Best rank:", best_match[1] print "\t", '"' + hd.data['hoerdat'].get_data('title') + '"'
def _configure(self): """ Set members based using inventory. """ Logger._configure(self) self.includeDealloc = self.inventory.includeDealloc return
def _add_player(self): """ Dodaje połączenie gracza do serwera. Wchodzi w to sprawdzanie, czy może się połączyć (serwer nie jest pełen), oraz odpowiednie zapisanie i rejestrację danych po stronie serwera. """ client_sock, _ = self._sock.accept() if len(self._client_sockets) == 2: client_sock.close() return client_sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) self._register_in_poll(client_sock) pl = 0 for key in self._player_bindings: if self._player_bindings[key] == 0: pl = 1 break self._player_bindings[client_sock.fileno()] = pl self._proto.push_player(client_sock.fileno(), pl+1) if len(self._client_sockets) == 2: Logger.log('Mamy dwóch graczy, startujemy grę!') self._gamestate.start() self._proto.push_start()
class BtObjectPersister: def __init__(self, persister, torrentIdent): self.persister = persister self.torrentIdent = torrentIdent self.log = Logger('BtPersister', '%-6s - ', torrentIdent) def load(self, key): return self.persister.load(self.torrentIdent+'-'+key) def get(self, key, default=None): return self.persister.get(self.torrentIdent+'-'+key, default) def store(self, key, obj, encoding='base64', sync=False): self.persister.store(self.torrentIdent+'-'+key, obj, encoding, sync) def remove(self, key, strict=False, sync=False): self.persister.remove(self.torrentIdent+'-'+key, strict, sync) def removeAll(self): btKeys = self.persister.keys('^'+self.torrentIdent+'-') for key in btKeys: self.log.debug('Removing key "%s" from persister', key) self.persister.remove(key)
class WikiCrawler(object): TAG = "WikiCrawler" def __init__(self): self.queue = QueueProxy() self.entryMgr = EntryManager() self.catagoryMgr = CatagoryManager() self.logger = Logger() def run(self): while True: try: sleep(1) url = self.queue.pop() if self.entryMgr.contains(url): self.entryMgr.update(url) else: html = HtmlReader(url).read() entry = HtmlParser(html, url).parse() self.entryMgr.add(entry) self.catagoryMgr.add(entry) for child in entry.children: self.queue.push(child) except Exception, e: self.logger.e(self.TAG, str(e))
def work(self, imageProcessor, filters): if not imageProcessor: raise Exception('No image processor for work') imageNumbersForProcess = self._getImageNumbersForProcess() Logger.write('end explode video. images count=%d' % len(imageNumbersForProcess)) imageProcessor.process(imageNumbersForProcess, filters)
class PointLabel: def __init__(self, df: pd.DataFrame, ax, fig, x_axis: str, y_axis: str, labels: list, file_=None): self.df = df.copy() self.axis = ax self.fig = fig self.x_axis = x_axis self.y_axis = y_axis self.labels = labels if 'score' in self.labels: self.labels.remove('score') self.has_written_title = False if file_ is not None: self.file_handler = file_ else: self.file_handler = Logger('point_label.log') def onpick(self, event): """ inspired by http://matplotlib.org/examples/event_handling/pick_event_demo.html """ ind = event.ind[0] row = self.df.iloc[ind] if not self.has_written_title: self.file_handler.log('picker %s %s %s description' % (self.x_axis, self.y_axis, '\t'.join(label for label in self.labels if label != 'description')), skip_stamp=True) self.has_written_title = True self.file_handler.log('picker %.2f\t%.2f\t%s %s' % (row[self.x_axis], row[self.y_axis], '\t'.join("%.2f" % row[label] for label in self.labels if label != 'description'), row['description']), skip_stamp=True)
def __init__(self): QtGui.QDialog.__init__(self) self.setupUi(self) self.progressBar.setFormat('%p%') self.progressBar.setValue(0) # ic=() #datafromsamson={'FAM': u'surname', 'IM': u'firstname', 'OT': u'parentname'} #todbf={} ImportConfig.parse() # samsmgr=SamsonManager() # samsmgr.ManageRawKmivcData(KmivcManager.Getrawkmivcdata()) #for n in range(0,50,1): # itme=QtGui.QListWidgetItem('test'+str(n)) # self.listWidget.addItem(itme) self.samsonadr.setText(ConnectionsManager.samsonconn['ip']) self.samsonbd.setText(ConnectionsManager.samsonconn['db']) self.samsonlogin.setText(ConnectionsManager.samsonconn['login']) self.samsonpassword.setText(ConnectionsManager.samsonconn['password']) #connect(lineEdit,SIGNAL(textChanged(QString)),this,SLOT(textChangedSlot(QString))); QtCore.QObject.connect(self.samsonadr, QtCore.SIGNAL("textChanged(QString)"), self.on_samsonadr_changed) QtCore.QObject.connect(self.samsonbd, QtCore.SIGNAL("textChanged(QString)"), self.on_samsonbd_changed) QtCore.QObject.connect(self.samsonlogin, QtCore.SIGNAL("textChanged(QString)"), self.on_samsonlogin_changed) QtCore.QObject.connect(self.samsonpassword, QtCore.SIGNAL("textChanged(QString)"), self.on_samsonpassword_changed) Logger.log(u'Программа запущена')
def step_impl(context, element_name): element = world.find_element(element_name) if element.wait_for_attribute('selected', False): return True else: log.failed("Verify element '%s' unchecked?" % element.name, "checked", "unchecked")
class RequestBodyReciever(Protocol): """needed to actually send the response from a server (because of the way the response object works). Passes any data it recieves to the Peer writer. This is an unfortunate side effect of the twisted architecture. A response object cannot pass it's body onwards without the use of this mitigating class""" def __init__(self,pClient,range): self.pClient = pClient #reference to persistent client class that holds an #open TCP connection with the peer self.recvd = 0 self.start = range[0] #placeholder for a deferred callback (incase one is eventually needed) self.size = range[1] - range[0] self.log = Logger() def repeatCallback(self): try: range = self.pClient.father.getNextChunk(self.pClient.id) if range != None: self.pClient.getChunk(range) except: self.log.warning('error in repeat callback on dlp') def dataReceived(self,bytes): self.recvd += len(bytes) self.pClient.father.appendData(self.pClient,self.start,bytes) def connectionLost(self,reason): if self.recvd < self.size: #server sent back a splash page or something other then the desired content self.pClient.father.endSession("Mismatched response length from server") self.log.info("Server ended transmission successfully with local pClient") self.repeatCallback()
def push_all(self, packet): """ Wrzuca na wszystkie kolejki pakietów odpowiedni pakiet do wysłania. """ Logger.log('Puszczamy pakiet WSZYSTKIM...') for client in self._clients: self._queues[client].append(packet)
def join(uniqueurl): Logger.printCurrentState( self.RequestHandler.SessionStorage.sessions) Logger.log('Received request on /joingame') res = self.RequestHandler.dispatchRequest( Request(requestKind=RequestKind.JoinGameRequest, uniqueurl=uniqueurl)) return str(res)
def __init__(self, x_init, y_init, mass, radius, borders): Logger.debug("KINEMATICS: PhysicsObject init(x_init=%s, y_init=%s, mass=%s, radius=%s, borders=%s)", str(x_init), str(y_init), str(mass), str(radius), str(borders)) self._pos = Vector(x_init, y_init) self._mass = mass self._radius = radius self._vel = Vector(0, 0) self._borders = borders
def step_impl(context): alert = world.get_alert_when_exist() if alert is not None: alert.dismiss() else: log.failed("Fail to cancel popup alert")
def step_impl(context): alert = world.get_alert_when_exist() if alert is not None: alert.accept() else: log.failed("Fail to accept popup alert")
def step_impl(context, element_name): element = world.find_element(element_name) if element.value == "": return True else: log.failed("Verify value is empty?", element.value, "<blank>")
def step_impl(context, element_name): element = world.find_element(element_name) if element.value == context.text: return True else: log.failed("Verify value is? (multi-line)", element.value, context.text)
def step_impl(context, element_name): element = world.find_element(element_name) if context.text in element.value: return True else: log.failed("Verify value contains? (multi-line)", element.value, value)
def step_impl(context, element_name): element = world.find_element(element_name) if element.wait_for_not_exist(): return True else: log.failed("Verify element '%s' exists?" % element.name, "exists", "not exists")
def step_impl(context, element_name): element = world.find_element(element_name) if element.wait_for_attribute('visible', True): return True else: log.failed("Verify element '%s' visible?" % element.name, "not visible", "visible")
def step_impl(context, element_name): element = world.find_element(element_name) if element.wait_for_attribute('enabled', False): return True else: log.failed("Verify element '%s' disabled?" % element.name, "enabled", "disabled")
def __main__(): config= Config() u_base_path = config.base_path logger = Logger(config.ftp_log_path) logger.set_debug_level('info') dupmerge = DupMerge( config.md5_db_path, u_base_path, {'logger': logger}, ) ftp = FTPFetch( host = config.ftp_hostname, port = config.ftp_port, credentials = config.ftp_credentials, params = { 'dstdir': config.ftp_base_path, 'db': config.ftp_db_path, 'ignore': config.ftp_ignore_list, 'possible_hidden_dirs': config.ftp_possible_hidden_dirs, 'dupmerge': dupmerge, 'reverse': 0, 'old_cleanup': 1, 'logger': logger, } ) ftp.link_all_duplicates()
def step_impl(context, element_name, value): value = world.replace_variables(value) element = world.find_element(element_name) if value in element.value: return True else: log.failed("Verify value contains?", element.value, value)
def step_impl(context, element_name, value): value = world.replace_variables(value) element = world.find_element(element_name) if element.get_attribute('title') == value: return True else: log.failed("Verify tooltip text is?", element.title, value)
def main(): start_time = time.perf_counter() # Setup logger log_to_console = '-p' in sys.argv logger = Logger('log.txt', log_to_console) # Get urls of the community api files from the api directory ff_api_urls = get_ff_api_urls(logger) # Get the urls to the nodelist and nodes_json files ff_urls = UrlSet() nodelist_urls = UrlSet() threads = [] for i in range(0, NUMBER_OF_THREADS): thread = Thread(target=get_map_urls, args=(ff_api_urls, logger, ff_urls, nodelist_urls)) threads.append(thread) thread.start() for thread in threads: thread.join() nodes_json_urls = get_nodes_json_urls(ff_urls) # Get the nodes from the nodes_json_urls nodes = NodeDict() threads = [] for i in range(0, NUMBER_OF_THREADS): thread = Thread(target=get_nodes_from_nodes_json_urls, args=(nodes_json_urls, nodes, logger)) threads.append(thread) thread.start() for thread in threads: thread.join() # Get the nodes from the nodelist_urls threads = [] for i in range(0, NUMBER_OF_THREADS): thread = Thread(target=get_nodes_from_nodelist_urls, args=(nodelist_urls, nodes, logger)) threads.append(thread) thread.start() for thread in threads: thread.join() # Write found nodes in a json file. data = dict() data['timestamp'] = int(time.time()) nodes_out = nodes.getNodes() logger.log(str(len(nodes_out)) + " nodes found.") data['nodes'] = nodes_out write_json_to_file(data) end_time = time.perf_counter() duration = end_time - start_time logger.log('Elapsed time: ' + str(duration))
class Cotorra( object ): def __init__(self): self.config = Configurator('/etc/core/etc/cotorra.cfg') self.logger = Logger(FILE_LOG) def main(self): stdin = sys.stdin.readlines() self.logger.addLine('Veo el correo')
def step_impl(context, element_name, value): value = world.replace_variables(value) element = world.find_element(element_name) classes = element.get_attribute('class') if value in classes: return True else: log.failed("Verify class contains?", classes, value)
def __init__(self, name="memory_logger"): """ Constructor. """ Logger.__init__(self, name) self.megabyte = float(2**20) self.memory = {} self.memory['Completion'] = 0 return
def __init__(self, image, imagerect, position): """ :param image: shape or image to be drawn :param position: position on screen (of type Point) :param imagerect: rectangle of image to be displayed """ Logger.debug("DRAWABLE: init(position=(%s)", str(position)) self._image = image self._pos = position
def __init__(self, datasetPath, testPath, dictionaryPath): self.datasetPath = datasetPath self.testPath = testPath self.dictionary = dictionaryPath self.logger = Logger(self.logLevel)
def main(): global args, best_prec1, logger args = parser.parse_args() os.makedirs(args.prefix) os.makedirs(os.path.join(args.prefix, 'tensorboard')) logger = Logger(os.path.join(args.prefix, 'tensorboard')) # create model if args.pretrained: print("=> using pre-trained model '{}'".format(args.arch)) model = models.__dict__[args.arch](pretrained=True) else: print("=> creating model '{}'".format(args.arch)) model = models.__dict__[args.arch]() if torch.cuda.is_available(): model = torch.nn.DataParallel(model).cuda() # define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss() if torch.cuda.is_available(): criterion = criterion.cuda() optimizer = torch.optim.SGD(model.parameters(), args.lr, momentum=args.momentum, weight_decay=args.weight_decay) # optionally resume from a checkpoint if args.resume: if os.path.isfile(args.resume): print("=> loading checkpoint '{}'".format(args.resume)) checkpoint = torch.load(args.resume) args.start_epoch = checkpoint['epoch'] best_prec1 = checkpoint['best_prec1'] model.load_state_dict(checkpoint['state_dict']) optimizer.load_state_dict(checkpoint['optimizer']) print("=> loaded checkpoint '{}' (epoch {})".format( args.resume, checkpoint['epoch'])) else: print("=> no checkpoint found at '{}'".format(args.resume)) cudnn.benchmark = True # Data loading code traindir = os.path.join(args.data, 'train') valdir = os.path.join(args.data, 'test') normalize = transforms.Normalize(mean=[0.4933, 0.5221, 0.5547], std=[0.2522, 0.2433, 0.2646]) train_dataset = datasets.ImageFolder( traindir, transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])) train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True, num_workers=args.workers, pin_memory=True) val_loader = torch.utils.data.DataLoader(datasets.ImageFolder( valdir, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])), batch_size=args.batch_size, shuffle=False, num_workers=args.workers, pin_memory=True) if args.evaluate: with torch.no_grad(): validate(val_loader, model, criterion) return for epoch in range(args.start_epoch, args.epochs): adjust_learning_rate(optimizer, epoch) # train for one epoch train(train_loader, model, criterion, optimizer, epoch) with torch.no_grad(): # evaluate on validation set prec1 = validate(val_loader, model, criterion) # remember best prec@1 and save checkpoint is_best = prec1 > best_prec1 best_prec1 = max(prec1, best_prec1) save_checkpoint( { 'epoch': epoch + 1, 'arch': args.arch, 'state_dict': model.state_dict(), 'best_prec1': best_prec1, 'optimizer': optimizer.state_dict(), }, is_best) # 混淆矩阵 compute_validate_meter(model, 'checkpoint.pth.tar', val_loader)
class RttScriptRunner_TriggerTest: """ Class to run any command within the RTT post-processing 'command': command to run (including any arguments) %TESTNAME is replaced with the jobDisplayName (or test name if not set) %JOBLOG is replaced with the RTT job log file %BRANCH is replaces with the release branch $<ENVVAR> is replaced with the environment variable (if exists) 'logfile': log file, if not given write to RTT logger """ def __init__(self, paramDict={}): from Logger import Logger self.success = 0 self.error = -1 self.logger = Logger() self.command = paramDict.get('command', None) self.logfile = paramDict.get( 'logfile', "test." + self.__class__.__name__ + ".log") jDescriptor = paramDict.get('JobDescriptor', None) if jDescriptor: testname = jDescriptor.jobDisplayName if testname == '': testname = jDescriptor.name self.command = self.command.replace("%TESTNAME", testname) self.command = self.command.replace("%JOBLOG", jDescriptor.log) self.command = self.command.replace("%BRANCH", jDescriptor.paths.branch) for k, v in os.environ.iteritems(): self.command = self.command.replace("$" + k, v) def run(self): if not self.command: self.logger.error("'command' parameter not set") return self.error self.logger.debug("ScriptRunner: Executing '%s'" % self.command) # Run command and get output import subprocess as sp proc = sp.Popen(self.command, shell=True, stdout=sp.PIPE, stderr=sp.STDOUT) stdout = proc.communicate()[0] # Write output to log file or RTT logger if self.logfile: f = open(self.logfile, "w") f.write(stdout) f.close() else: for l in stdout.splitlines(True): self.logger.info(l.rstrip('\n')) if proc.returncode != 0: return self.error else: return self.success
class MainTab: """ This class handles the logic of the main tab """ #: The tab specific logger logger = Logger(Strings.mainTabLoggerName).getLogger() #: Statusbar labels statusBarApplicationStatus = None #: These text will appear in the status bar statusBarActiveStatuses = [] statusBarInterface = None statusBarProject = None ___playing = False @staticmethod def addApplicationStatus(status): """ Add a new status to the status bar by name (ordered). If no status is present, it will display ``Strings.statusBarReady`` :param status: The new status to add """ if status is not None and status != Strings.statusBarReady: if status not in MainTab.statusBarActiveStatuses: MainTab.statusBarActiveStatuses.append(status) # Sort to keep order MainTab.statusBarApplicationStatus.setText(", ".join( sorted(MainTab.statusBarActiveStatuses)).title()) else: MainTab.statusBarApplicationStatus.setText(Strings.statusBarReady) @staticmethod def removeApplicationStatus(status): """ Remove a status from the status bar. For statuses with multiple possible values (e.g. ``Sending (X Threads)`` the search will be done using a substring search :param status: The status to remove :return: """ if status is not None: if status in MainTab.statusBarActiveStatuses: MainTab.statusBarActiveStatuses.remove(status) # For sender threads: search via substring else: # If a substring search matches get the element matchingStatus = None matchingStatusList = [ value for value in MainTab.statusBarActiveStatuses if status in value ] if len(matchingStatusList) > 0: matchingStatus = matchingStatusList[0] if matchingStatus is not None: MainTab.statusBarActiveStatuses.remove(matchingStatus) if len(MainTab.statusBarActiveStatuses) == 0: MainTab.statusBarApplicationStatus.setText( Strings.statusBarReady) else: MainTab.statusBarApplicationStatus.setText(", ".join( sorted(MainTab.statusBarActiveStatuses)).title()) @staticmethod def setGlobalInterfaceStatus(red=False): """ Sets the text of the global interface status in the status bar. If the global CANData instance is None then the text will read "None". :param red: Optional; If this is set to True, the text will appear red. Else black. """ if red: MainTab.statusBarInterface.setStyleSheet("QLabel { color : red; }") else: MainTab.statusBarInterface.setStyleSheet( "QLabel { color : black; }") if Globals.CANData is not None: MainTab.statusBarInterface.setText("Global interface: " + Globals.CANData.toString()) else: MainTab.statusBarInterface.setText("Global interface: None") @staticmethod def setProjectStatus(projectName, red=False): """ Sets the text of the project status in the status bar. :param projectName: The text to put as the new project name :param red: Optional; If this is set to True, the text will appear red. Else black. """ if not red: MainTab.statusBarProject.setStyleSheet("QLabel { color : black; }") else: MainTab.statusBarProject.setStyleSheet("QLabel { color : red; }") MainTab.statusBarProject.setText("Project: " + projectName) @staticmethod def populateProjects(keepCurrentIndex=False): """ This populates the project ComboBox in the main tab. :param keepCurrentIndex: If this is set to True, the previously selected index will be re-selected in the end """ # Save the index if keepCurrentIndex: currentIndex = Globals.ui.comboBoxProjectSet.currentIndex() projectComboBox = Globals.ui.comboBoxProjectSet projectComboBox.clear() projects = Globals.db.getProjects() for i in range(len(projects)): projectComboBox.addItem(projects[i].toComboBoxString()) projectComboBox.setItemData(i, projects[i]) # If theres only 1 project then set it as active # if len(projects) == 1: # Globals.project = MainTab.setProject() if keepCurrentIndex: # Set the remembered index Globals.ui.comboBoxProjectSet.setCurrentIndex(currentIndex) @staticmethod def setProject(wasDeleted=False, setNone=False): """ This sets the current project to the currently selected project in the corresponding ComboBox. Also, the status bar and project specific ComboBoxes and GUI Elements will be updated. :param wasDeleted: This is set to True if the current selected project was deleted. This causes ``Globals.project`` to become None, too. :param wasNull: This is set to True, if the project has to be set to None. Default: False """ if not wasDeleted and not setNone: Globals.project = Globals.ui.comboBoxProjectSet.itemData( Globals.ui.comboBoxProjectSet.currentIndex()) if Globals.project is None: return MainTab.setProjectStatus(Globals.project.name) elif setNone: Globals.project = None MainTab.setProjectStatus(Strings.statusBarNoProject, red=True) else: Globals.project = None MainTab.setProjectStatus(Strings.statusBarProjectWasDeleted, red=True) MainTab.logger.info(Strings.mainTabLoadingProjectData) # Update the project data that is managed Globals.managerTabInstance.getKnownPacketsForCurrentProject() Globals.managerTabInstance.populatePacketSets() Globals.managerTabInstance.populateKnownPackets() Toolbox.Toolbox.toggleDisabledProjectGUIElements() MainTab.logger.info( Strings.mainTabProjectSet + " " + Globals.project.name if Globals.project is not None else "None") @staticmethod def loadKernelModules(): """ Load kernel modules to interact with CAN networks (``can`` and ``vcan``). """ cmds = "modprobe can; modprobe vcan" output, error = Toolbox.Toolbox.runRootshell(cmds) @staticmethod def easterEgg(event): """ Nothing to see here :return: fun """ if Strings.musicFilePath in Toolbox.Toolbox.mp3Processes: Toolbox.Toolbox.stopMP3(Strings.musicFilePath) else: Toolbox.Toolbox.playMP3(Strings.musicFilePath) @staticmethod def addVCANInterface(): """ Manually add a virtual CAN interface. This uses a syscall to ``ip link``. If this call succeeds, a new CANDataInstance will be created using :func:`~src.CANData.createCANDataInstance`. The detected CAN interfaces will be refreshed, too. """ vifaceName = "vcan" + str(Globals.ui.spinBoxVCANIndex.value()) cmds = [ "ip link add dev " + vifaceName + " type vcan", "ip link set up " + vifaceName ] for cmd in cmds: output, error = Toolbox.Toolbox.runRootshell(cmd) if error is not None and error.decode("utf-8") == "": CANData.createCANDataInstance(vifaceName) MainTab.logger.info(Strings.mainTabVCANAdded + " " + vifaceName) else: MainTab.logger.warn(error.decode("utf-8")) MainTab.detectCANInterfaces() @staticmethod def removeVCANInterface(): """ This removes the currently selected VCAN interface. This uses a syscall to ``ip link``. If the removed interface was the current global interface, the global interface will become None. :return: """ vifaceName = "vcan" + str(Globals.ui.spinBoxVCANIndex.value()) wasGlobalInterface = False if Globals.CANData is not None and vifaceName == Globals.CANData.ifaceName: wasGlobalInterface = True # Interface still used or not present -- abort if not CANData.deleteCANDataInstance(vifaceName): return cmd = "ip link delete " + vifaceName output, error = Toolbox.Toolbox.runRootshell(cmd) SnifferTab.SnifferTab.removeSniffer(snifferTabName=vifaceName) MainTab.detectCANInterfaces() Toolbox.Toolbox.updateInterfaceLabels() Toolbox.Toolbox.toggleDisabledSenderGUIElements() if error.decode("utf-8") == "": MainTab.logger.info(Strings.mainTabVCANRemoved + " " + vifaceName) if wasGlobalInterface: Globals.CANData = None else: MainTab.logger.warn(error.decode("utf-8")) MainTab.setGlobalInterfaceStatus(red=True) @staticmethod def updateVCANButtons(): """ Update the text of the buttons to add and remove VCAN interfaces. """ currentVal = str(Globals.ui.spinBoxVCANIndex.value()) Globals.ui.buttonVCANAdd.setText("Add vcan" + currentVal) Globals.ui.buttonVCANRemove.setText("Remove vcan" + currentVal) @staticmethod def detectCANInterfaces(updateLabels=True): """ Detect CAN and VCAN interfaces available in the system. A syscall to ``/sys/class/net`` is being used for this. For every detected interface a new CANData instance will be created using :func:`~src.CANData.createCANDataInstance`. Also, interface labels and the global interface ComboBox will be updated. :param updateLabels: Whether to update the interface labels or not """ ifaces = sorted(os.listdir("/sys/class/net")) CANIfaces = [] for iface in ifaces: if "can" in iface: CANIfaces.append(iface) removedInterfaceNames = CANData.rebuildCANDataInstances(CANIfaces) for removedInterfaceName in removedInterfaceNames: SnifferTab.SnifferTab.removeSniffer( snifferTabName=removedInterfaceName) Toolbox.Toolbox.populateInterfaceComboBox(Globals.ui.comboBoxInterface, reselectCurrentItem=False) Toolbox.Toolbox.updateCANDataInstances( CANData.getGlobalOrFirstInstance()) for CANIface in CANIfaces: SnifferTab.SnifferTab.addSniffer(CANIface) if updateLabels: Toolbox.Toolbox.updateInterfaceLabels() @staticmethod def preselectUseBitrateCheckBox(): """ Preselect the VCAN CheckBox state because we can't use the bitrate along with VCAN interfaces. """ currentInterface = Globals.ui.comboBoxInterface.itemData( Globals.ui.comboBoxInterface.currentIndex()) if currentInterface is None: return if currentInterface.VCAN: Globals.ui.checkBoxMainUseVCAN.setChecked(True) else: Globals.ui.checkBoxMainUseVCAN.setChecked(False) MainTab.VCANCheckboxChanged() @staticmethod def applyGlobalInterfaceSettings(): """ Set the currently selected interface as the global interface. Also, the bitrate will be updated and GUI elements will be toggled. The CANData instances of all **inactive** tabs will also be set to the global interface. """ selectedInterfaceName = Globals.ui.comboBoxInterface.currentText() if len(selectedInterfaceName) == 0: return selectedBitrate = Globals.ui.spinBoxBitrate.value() selectedCANData = CANData.CANDataInstances[selectedInterfaceName] # Set all inactive interfaces to the global interface Toolbox.Toolbox.updateCANDataInstances(selectedCANData) Globals.CANData = selectedCANData MainTab.setGlobalInterfaceStatus() # It's still active, we set it but won't save the settings though if selectedCANData.active: MainTab.logger.info(Strings.activeCANDataWontSave + selectedCANData.ifaceName) Globals.ui.spinBoxBitrate.setValue(selectedCANData.bitrate) Globals.ui.checkBoxMainUseVCAN.setChecked( QtCore.Qt.Checked if selectedCANData.VCAN else QtCore.Qt. Unchecked) return # It's not active, updated VCAN and the bitrate else: selectedCANData.VCAN = Globals.ui.checkBoxMainUseVCAN.isChecked() if selectedCANData.updateBitrate(selectedBitrate): Globals.ui.buttonApplyInterface.setStyleSheet( "background-color: green") else: Globals.ui.buttonApplyInterface.setStyleSheet( "background-color: red") return MainTab.logger.info(Strings.mainTabCANConfigUpdated) # Enable all disabled buttons because an interface was set Toolbox.Toolbox.toggleDisabledSenderGUIElements() # Update all labels to the new global interface settings Toolbox.Toolbox.updateInterfaceLabels() @staticmethod def applyLogLevelSetting(): """ Set the minimum logging level to display messages for. """ selectedLevel = Globals.ui.comboBoxLoglevel.currentText() if selectedLevel == "INFO": Logger.minLogLevel = logging.INFO elif selectedLevel == "DEBUG": Logger.minLogLevel = logging.DEBUG elif selectedLevel == "WARNING": Logger.minLogLevel = logging.WARN elif selectedLevel == "ERROR": Logger.minLogLevel = logging.ERROR elif selectedLevel == "CRITICAL": Logger.minLogLevel = logging.CRITICAL MainTab.logger.debug(Strings.mainTabLogLevelChanged) @staticmethod def setupStatusBar(): """ Add labels to the status bar and prepare it. """ MainTab.statusBarApplicationStatus = QLabel(Strings.statusBarReady, Globals.ui) MainTab.statusBarInterface = QLabel( Strings.statusBarSelectGlobalInterface, Globals.ui) MainTab.statusBarProject = QLabel(Strings.statusBarNoProject, Globals.ui) MainTab.setProjectStatus(Strings.statusBarNoProject) MainTab.statusBarApplicationStatus.setFrameStyle(QFrame.Panel | QFrame.Sunken) MainTab.statusBarInterface.setFrameStyle(QFrame.Panel | QFrame.Sunken) MainTab.statusBarProject.setFrameStyle(QFrame.Panel | QFrame.Sunken) MainTab.statusBarApplicationStatus.setStyleSheet( "QLabel { color : black; }") MainTab.statusBarInterface.setStyleSheet("QLabel { color : red; }") MainTab.statusBarProject.setStyleSheet("QLabel { color : red; }") Globals.ui.statusBar().addPermanentWidget( MainTab.statusBarApplicationStatus, 1) Globals.ui.statusBar().addPermanentWidget(MainTab.statusBarInterface, 2) Globals.ui.statusBar().addPermanentWidget(MainTab.statusBarProject, 3) @staticmethod def prepareUI(): """ 1. Setup the status bar 2. Detect CAN interfaces and preselect the VCAN CheckBox 3. Populate project ComboBoxes 4. Add the logo """ MainTab.setupStatusBar() MainTab.detectCANInterfaces() MainTab.populateProjects() MainTab.preselectUseBitrateCheckBox() # Setup the "fork me" ribbon pixmapLogo = QPixmap(Settings.ICON_PATH) Globals.ui.labelMainLogo.setPixmap(pixmapLogo) @staticmethod def VCANCheckboxChanged(): """ Clickhandler for the VCAN CheckBox which causes the SpinBox to be toggled. """ if Globals.ui.checkBoxMainUseVCAN.isChecked(): Globals.ui.spinBoxBitrate.setEnabled(False) else: Globals.ui.spinBoxBitrate.setEnabled(True)
def init_with_three_parameters(self, n_labels_pass, train_my_dataset, sub_zone_value="-1"): if not sub_zone_value == "-1": self.sub_lable_name = "H" + str(sub_zone_value) logger = Logger.set_logger() ranks = train_my_dataset.get_ranges() self.n_variables = train_my_dataset.get_ninputs() self.names = train_my_dataset.get_names() # int array self.nlabels_array = [0 for x in range(self.n_variables)] # bool array self.varreal_array = [True for x in range(self.n_variables)] self.database = [[Fuzzy() for y in range(n_labels_pass)] for x in range(self.n_variables)] self.database_ini = [[Fuzzy() for y in range(n_labels_pass)] for x in range(self.n_variables)] for i in range(0, self.n_variables): rank = float(abs(float(ranks[i][1]) - float(ranks[i][0]))) self.varreal_array[i] = False if train_my_dataset.is_nominal(i): self.nlabels_array[i] = int(rank) + 1 elif train_my_dataset.is_integer(i) and (rank + 1) <= n_labels_pass: self.nlabels_array[i] = int(rank) + 1 else: self.nlabels_array[i] = n_labels_pass self.varreal_array[i] = True self.database[i] = [Fuzzy() for x in range(self.nlabels_array[i])] self.database_ini[i] = [ Fuzzy() for x in range(self.nlabels_array[i]) ] if (self.nlabels_array[i] - 1.0) == 0: mark = 0 else: mark = float(rank / float(self.nlabels_array[i] - 1.0)) for j in range(0, self.nlabels_array[i]): self.database[i][j] = Fuzzy() self.database_ini[i][j] = Fuzzy() value = float(ranks[i][0]) + mark * (j - 1) self.database_ini[i][j].x0 = self.database[i][ j].x0 = self.set_value(value, ranks[i][0], ranks[i][1]) value = float(ranks[i][0]) + mark * j self.database_ini[i][j].x1 = self.database[i][ j].x1 = self.set_value(value, ranks[i][0], ranks[i][1]) value = float(ranks[i][0]) + mark * (j + 1) self.database_ini[i][j].x3 = self.database[i][ j].x3 = self.set_value(value, ranks[i][0], ranks[i][1]) self.database_ini[i][j].y = self.database[i][j].y = 1.0 if not sub_zone_value == "-1": self.database[i][ j].name = "L_" + self.sub_lable_name + "_" + str( j) + "(" + str(self.nlabels_array[i]) + ")" self.database_ini[i][ j].name = "L_" + self.sub_lable_name + "_" + str( j) + "(" + str(self.nlabels_array[i]) + ")" else: self.database[i][j].name = "L_" + str(j) + "(" + str( self.nlabels_array[i]) + ")" self.database_ini[i][j].name = "L_" + str(j) + "(" + str( self.nlabels_array[i]) + ")" """
from myTime import * # Read configuration file config = ConfigParser() config.readfp(open(FULL_MAIN_CONF)) logname = config.get('CS', 'logname') # Full name for the logfile log_level = config.get('CS', 'log_level') # Level of logging # If not defined in configuration file, set defaults if (not logname): logname = CS + "/log/" + "CS.log" if (not log_level): log_level = "DEBUG" # Enable logging logger = Logger(logname, log_level, "CS") logger = logger.getLogger() form = cgi.FieldStorage() clientName = form["clientName"].value host = form["host"].value machinesString = form["machinesString"].value logger.info("Execution pdsGetClientListing.py for: " + clientName) if (form.has_key("glob")): glob = form["glob"].value else: glob = ".*" if (form.has_key("endDate")):
from GA import GA from TravelSalesPerson import TravelSalesPerson from Logger import Logger import numpy as np import datetime log = Logger('C-PR2392-10.log', level='info') # set the name of the log CROSS_RATE = 0.1 # crossover probability MUTATE_RATE = 0.02 # mutation probability POP_SIZE = 10 # the size of population N_GENERATIONS = 20000 # generations, that is, the number of iterations DATA_SET = 'PR2392' # the input city data, chosen from the ten data sets if __name__ == '__main__': starttime = datetime.datetime.now() env = TravelSalesPerson(DATA_SET) # get the data of all cities N_CITIES = env.N_CITIES ga = GA(DNA_size=N_CITIES, cross_rate=CROSS_RATE, mutation_rate=MUTATE_RATE, pop_size=POP_SIZE) # initialize the population print( 'Completion of population generation. Population Size: {}. DNA Length of Each Individual: {}' .format(ga.pop_size, ga.DNA_size)) for generation in range(N_GENERATIONS): lx, ly = ga.translateDNA( ga.pop, env.city_position) # get the DNA of the population fitness, total_distance = ga.get_fitness( lx, ly) # get the fitness and the travel distance of the population
config = get_config() prepare_data(os.path.join(config.data, 'train.txt'), views=config.num_views, shuffle=True) prepare_data(os.path.join(config.data, 'test.txt'), views=config.num_views, shuffle=False) caffe.set_device(0) caffe.set_mode_gpu() data_size = get_dataset_size(config, 'trainrotnet') prepare_solver_file(data_size=data_size) solver = caffe.get_solver(config.solver) if not config.test: LOSS_LOGGER = Logger("{}_loss".format(config.name)) ACC_LOGGER = Logger("{}_acc".format(config.name)) train(config, solver) config = add_to_config(config, 'weights', highest_model_saved) config = add_to_config(config, 'test', True) snapshot = get_highest_model(config) solver.restore(snapshot) caffemodel = os.path.splitext(snapshot)[0] + '.caffemodel' solver.net.copy_from(caffemodel) solver.test_nets[0].copy_from(caffemodel) print('Model restored') eval(config, solver)
def main(): parser = ArgumentParser() parser.add_argument("-s", "--spec-path", dest="specPath", default="../../SPECS") parser.add_argument("-x", "--source-path", dest="sourcePath", default="../../stage/SOURCES") parser.add_argument("-r", "--rpm-path", dest="rpmPath", default="../../stage/RPMS") parser.add_argument("-i", "--install-package", dest="installPackage", default=False, action="store_true") parser.add_argument("-p", "--publish-RPMS-path", dest="publishRPMSPath", default="../../stage/PUBLISHRPMS") parser.add_argument("-e", "--publish-XRPMS-path", dest="publishXRPMSPath", default="../../stage/PUBLISHXRPMS") parser.add_argument("-l", "--log-path", dest="logPath", default="../../stage/LOGS") parser.add_argument("-y", "--log-level", dest="logLevel", default="error") parser.add_argument("-z", "--top-dir-path", dest="topDirPath", default="/usr/src/photon") parser.add_argument("-b", "--build-root-path", dest="buildRootPath", default="/mnt") parser.add_argument("-t", "--threads", dest="buildThreads", default=1, type=int, help="Number of working threads") parser.add_argument("-m", "--tool-chain-stage", dest="toolChainStage", default="None") parser.add_argument("-c", "--pullsources-config", dest="pullsourcesConfig", default="pullsources.conf") parser.add_argument("-d", "--dist-tag", dest="dist", default="") parser.add_argument("-k", "--input-RPMS-path", dest="inputRPMSPath", default=None) parser.add_argument("-n", "--build-number", dest="buildNumber", default="0000000") parser.add_argument("-v", "--release-version", dest="releaseVersion", default="NNNnNNN") parser.add_argument("-u", "--enable-rpmcheck", dest="rpmCheck", default=False, action="store_true") parser.add_argument("-a", "--source-rpm-path", dest="sourceRpmPath", default="../../stage/SRPMS") parser.add_argument("-w", "--pkginfo-file", dest="pkgInfoFile", default="../../stage/pkg_info.json") parser.add_argument("-g", "--pkg-build-option-file", dest="pkgBuildOptionFile", default="../../common/data/pkg_build_options.json") parser.add_argument("-q", "--rpmcheck-stop-on-error", dest="rpmCheckStopOnError", default=False, action="store_true") parser.add_argument("-bd", "--publish-build-dependencies", dest="publishBuildDependencies", default=False) parser.add_argument("-pw", "--package-weights-path", dest="packageWeightsPath", default=None) parser.add_argument("-bt", "--build-type", dest="pkgBuildType", choices=['chroot', 'container'], default="chroot") parser.add_argument("-F", "--kat-build", dest="katBuild", default=None) parser.add_argument("-pj", "--packages-json-input", dest="pkgJsonInput", default=None) parser.add_argument("PackageName", nargs='?') options = parser.parse_args() cmdUtils = CommandUtils() if not os.path.isdir(options.logPath): cmdUtils.runCommandInShell("mkdir -p " + options.logPath) logger = Logger.getLogger("Main", options.logPath, options.logLevel) errorFlag = False package = None pkgInfoJsonFile = options.pkgInfoFile if not os.path.isdir(options.sourcePath): logger.error("Given Sources Path is not a directory:" + options.sourcePath) errorFlag = True if not os.path.isdir(options.specPath): logger.error("Given Specs Path is not a directory:" + options.specPath) errorFlag = True if not os.path.isdir(options.publishRPMSPath): logger.error("Given RPMS Path is not a directory:" + options.publishRPMSPath) errorFlag = True if not os.path.isdir(options.publishXRPMSPath): logger.error("Given X RPMS Path is not a directory:" + options.publishXRPMSPath) errorFlag = True if not os.path.isdir(options.publishRPMSPath + "/" + platform.machine()): logger.error("Given RPMS Path is missing " + platform.machine() + " sub-directory:" + options.publishRPMSPath) errorFlag = True if not os.path.isdir(options.publishXRPMSPath + "/" + platform.machine()): logger.error("Given X RPMS Path is missing " + platform.machine() + " sub-directory:" + options.publishXRPMSPath) errorFlag = True if not os.path.isdir(options.publishRPMSPath + "/noarch"): logger.error("Given RPMS Path is missing noarch sub-directory:" + options.publishRPMSPath) errorFlag = True if not os.path.isdir(options.publishXRPMSPath + "/noarch"): logger.error("Given X RPMS Path is missing noarch sub-directory:" + options.publishXRPMSPath) errorFlag = True if options.inputRPMSPath is not None and not os.path.isdir( options.inputRPMSPath): logger.error("Given input RPMS Path is not a directory:" + options.inputRPMSPath) errorFlag = True if options.packageWeightsPath is not None and not os.path.isfile( options.packageWeightsPath): logger.error("Given input Weights file is not a file:" + options.packageWeightsPath) errorFlag = True if options.pkgJsonInput is not None and not os.path.isfile( options.pkgJsonInput): logger.error("Given input packages file is not a file:" + options.pkgJsonInput) errorFlag = True if options.installPackage: if not options.PackageName: logger.error("Please provide package name") errorFlag = True else: package = options.PackageName if errorFlag: logger.error( "Found some errors. Please fix input options and re-run it.") return False if not os.path.isdir(options.rpmPath): cmdUtils.runCommandInShell("mkdir -p " + options.rpmPath + "/" + platform.machine()) cmdUtils.runCommandInShell("mkdir -p " + options.rpmPath + "/noarch") if not os.path.isdir(options.sourceRpmPath): cmdUtils.runCommandInShell("mkdir -p " + options.sourceRpmPath) if not os.path.isdir(options.buildRootPath): cmdUtils.runCommandInShell("mkdir -p " + options.buildRootPath) logger.debug("Source Path :" + options.sourcePath) logger.debug("Spec Path :" + options.specPath) logger.debug("Rpm Path :" + options.rpmPath) logger.debug("Log Path :" + options.logPath) logger.debug("Log Level :" + options.logLevel) logger.debug("Top Dir Path :" + options.topDirPath) logger.debug("Publish RPMS Path :" + options.publishRPMSPath) logger.debug("Publish X RPMS Path :" + options.publishXRPMSPath) if options.installPackage: logger.debug("Package to build:" + package) get_packages_with_build_options(options.pkgBuildOptionFile) try: constants.setSpecPath(options.specPath) constants.setSourcePath(options.sourcePath) constants.setRpmPath(options.rpmPath) constants.setSourceRpmPath(options.sourceRpmPath) constants.setTopDirPath(options.topDirPath) constants.setLogPath(options.logPath) constants.setLogLevel(options.logLevel) constants.setDist(options.dist) constants.setBuildNumber(options.buildNumber) constants.setReleaseVersion(options.releaseVersion) constants.setPrevPublishRPMRepo(options.publishRPMSPath) constants.setPrevPublishXRPMRepo(options.publishXRPMSPath) constants.setBuildRootPath(options.buildRootPath) constants.setPullSourcesURL(get_baseurl(options.pullsourcesConfig)) constants.setInputRPMSPath(options.inputRPMSPath) constants.setRPMCheck(options.rpmCheck) constants.setRpmCheckStopOnError(options.rpmCheckStopOnError) constants.setPublishBuildDependencies(options.publishBuildDependencies) constants.setPackageWeightsPath(options.packageWeightsPath) constants.setKatBuild(options.katBuild) constants.initialize() # parse SPECS folder SPECS() if options.toolChainStage == "stage1": pkgManager = PackageManager() pkgManager.buildToolChain() elif options.toolChainStage == "stage2": pkgManager = PackageManager() pkgManager.buildToolChainPackages(options.buildThreads) elif options.installPackage: buildSpecifiedPackages([package], options.buildThreads, options.pkgBuildType) elif options.pkgJsonInput: buildPackagesInJson(options.pkgJsonInput, options.buildThreads, options.pkgBuildType, pkgInfoJsonFile, logger) else: buildPackagesForAllSpecs(options.buildThreads, options.pkgBuildType, pkgInfoJsonFile, logger) except Exception as e: logger.error("Caught an exception") logger.error(str(e)) # print stacktrace traceback.print_exc() sys.exit(1) sys.exit(0)
def modus_ponens(A,B): if A == B[1:len(A)+1]: Logger(B[len(A)+2:len(B)-1], 'Modus ponens for '+A+' and '+B) return B[len(A)+2:len(B)-1]
RCK = 0x20 CTR = 0x00 SDA = 0x1c RDA = 0x12 CLK = 0x18 if bmi + bmo != 1: raise ValueError("must specify one of 'bmi' or 'bmo' at cmd line") for x in (RCK, CTR, SDA, RDA, CLK): assert 0 <= x <= 255 ######################################################## print "connecting to ccu prog %s:%i" % (fechost, fecport) ccu = SimpleSocket(fechost, fecport) log = Logger() print print 'not sending cratereset, reset, piareset all as old script did' tag="FEC %d ring 0x%i :"%(fecslot,fecring) print 'tag is', tag fecringfound=False fecfound=False status="" for l in ccu.send("mapccu").readlines(): print l if l.strip().startswith("FEC %d"%(fecslot)): fecfound=True if l.strip().startswith(tag): fecringfound=True status=l.strip().split(":")[1].strip()
def A3(F,G): P = IMPLICATION(IMPLICATION(NOT(G),NOT(F)),IMPLICATION(IMPLICATION(NOT(G),F),G)) Logger(P,'A3 for '+F+' and '+G) return '('+'('+'('+NEG+G+')'+IMP+'('+NEG+F+')'+')'+IMP+'('+'('+'('+NEG+G+')'+IMP+F+')'+IMP+G+')'+')'
def A2(F,G,H): P = IMPLICATION(IMPLICATION(F,IMPLICATION(G,H)),IMPLICATION(IMPLICATION(F,G),IMPLICATION(F,H))) Logger(P,'A2 for '+F+' and '+G+' and '+H) return P
class ACL(QObject): updateError = pyqtSignal(name='updateError') update = pyqtSignal() downloadActiveUpdate = pyqtSignal() @pyqtProperty(str, notify=update) def hash(self): self.mutex.lock() h = self._hash self.mutex.unlock() return h @pyqtProperty(int, notify=update) def numRecords(self): self.mutex.lock() n = self._numRecords self.mutex.unlock() return n @pyqtProperty(int, notify=update) def numActiveRecords(self): self.mutex.lock() n = self._numActiveRecords self.mutex.unlock() return n @pyqtProperty(str, notify=update) def date(self): self.mutex.lock() d = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(self._date)) self.mutex.unlock() return d @pyqtProperty(str, notify=update) def status(self): self.mutex.lock() s = self._status self.mutex.unlock() return s @pyqtProperty(bool, notify=downloadActiveUpdate) def downloadActive(self): self.mutex.lock() isdl = self._downloadActive self.mutex.unlock() return isdl @downloadActive.setter def downloadActive(self, value): self.mutex.lock() self._downloadActive = value self.mutex.unlock() self.downloadActiveUpdate.emit() def __init__(self, loglevel='WARNING', netWorker=None, url='', cacheFile=None): QObject.__init__(self) self.logger = Logger(name='ratt.acl') self.logger.setLogLevelStr(loglevel) self.debug = self.logger.isDebug() if netWorker is None: self.logger.error('must set netWorker') exit(-1) self.netWorker = netWorker self.url = url self.cacheFile = cacheFile self.mutex = QMutex() self._acl = json.loads('[]') self._hash = '' self._numRecords = 0 self._numActiveRecords = 0 self._downloadActive = False self._date = int(time.time()) self._status = 'initialized' self.loadFile(self.cacheFile) def loadFile(self, filename=None): self.logger.info('loading ACL file %s' % filename) if filename is not None: f = QFile(filename) if not f.open(QIODevice.ReadOnly | QIODevice.Text): self.logger.error('error opening ACL file %s for read' % filename) return False bytes = f.readAll() if bytes.isEmpty(): self.logger.error('unabled to read from ACL file %s' % filename) f.close() return False f.close() info = QFileInfo(filename) modified = int(info.lastModified().toMSecsSinceEpoch() / 1000) return self.parseJSON(doc=str(bytes), save=False, date=modified, status='loaded_from_file') return False def saveFile(self, filename=None): self.logger.info('saving ACL file %s' % filename) if filename is not None: self.mutex.lock() doc = json.dumps(self._acl) self.mutex.unlock() f = QFile(filename) if not f.open(QIODevice.WriteOnly): self.logger.error('error opening ACL file %s for write' % filename) return False if f.write(str(doc)) == -1: f.close() self.logger.error('unabled to write to ACL file %s' % filename) return False f.close() return True return False @pyqtSlot() def download(self): if not self.downloadActive: self.logger.info('downloading ACL from ' + self.url) self.reply = self.netWorker.get(url=QUrl(self.url)) self.reply.finished.connect(self.slotDownloadFinished) self.downloadActive = True return True else: self.logger.warning('already busy downloading ACL') return False def slotDownloadFinished(self): self.logger.info('ACL download finished') error = self.reply.error() if error == QNetworkReply.NoError: self.logger.debug('no error, parsing response') self.parseJSON(doc=str(self.reply.readAll()), save=True, status='downloaded') else: self.logger.error('NetWorker response error: %s (%s)' % (error, self.reply.errorString())) self.reply.deleteLater() self.downloadActive = False def search(self, hash): self.mutex.lock() found_record = [] for record in self._acl: if record['tagid'] == hash: found_record = record break self.mutex.unlock() return found_record def countActive(self, j): active = 0 for record in j: if record['allowed'] == 'allowed': active = active + 1 return active def calcHash(self, j): m = hashlib.sha224() m.update(str(j).encode()) hash = m.hexdigest() return hash def parseJSON(self, doc, save=False, date=None, status=''): try: parsed = json.loads(doc) count = len(parsed) active = self.countActive(parsed) hash = self.calcHash(parsed) self.logger.debug('parseJSON hash=%s stored hash=%s' % (hash, self._hash)) if hash != self._hash: self.mutex.lock() self._acl = parsed self._numRecords = count self._numActiveRecords = active self._hash = hash if date is None: self._date = int(time.time()) else: self._date = date self._status = status self.mutex.unlock() self.logger.info( 'updated ACL with %d entries, %d active, hash=%s' % (self._numRecords, self._numActiveRecords, self._hash)) self.update.emit() if save: self.saveFile(self.cacheFile) return True else: self.mutex.lock() if date is None: self._date = int(time.time()) self._status = 'same_hash' self.mutex.unlock() self.update.emit() self.logger.error('no ACL update because same hash=%s' % (self._hash)) except: self.logger.exception('json parse error') self.updateError.emit() return False
from Logger import Logger from utils.FileUtils import FileUtils def get_root_path(): """ get project root path :return: root path """ fileRoot = getattr(sys.modules['__main__'], '__file__') return os.path.abspath(os.path.dirname(fileRoot)) # load banner # banner = os.path.join(get_root_path(), 'banner.txt') Logger.info('load banner from %s', banner) if FileUtils.exists(banner): file = open(file=banner) print(file.read()) file.close() else: Logger.info('banner not exists, skip it') # load configuration # Logger.info('start load default configuration file') config = os.path.join(get_root_path(), 'configuration', 'sql-view.yaml') Logger.info('load configuration from %s', config) try: file = open(config, 'r') Configuration = yaml.load(file, Loader=yaml.FullLoader) except Exception as ex:
reload(sys) sys.setdefaultencoding('utf8') #以上三行代码放处理中文的代码前面 ################################################################################################# #logging.basicConfig(level=logging.DEBUG,format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',datefmt='%a, %d %b %Y %H:%M:%S', filename='myapp.log',filemode='w') #定义一个StreamHandler,将INFO级别或更高的日志信息打印到标准错误,并将其添加到当前的日志处理对象# #console = logging.StreamHandler() #console.setLevel(logging.INFO) #formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s') #console.setFormatter(formatter) #logging.getLogger('').addHandler(console) ################################################################################################# #日志工具 logger = Logger('log/all.log', level='debug').logger wechat = {} wechat['window_x'] = 600 #微信聊天窗口打字位置 x坐标 wechat['window_y'] = 600 #微信聊天窗口打字位置 y坐标 wechat['content_x'] = 680 #微信URL x坐标 wechat['content_y'] = 450 #微信URL y坐标 wechat['window_web_x'] = 816 #微信浏览器关闭位置 x坐标 wechat['window_web_y'] = 12 #微信聊天窗口打字位置 y坐标 URL = "collect_wechat_run_usersub_pc.php?web=2" URL_IP = "shell.myzaker.com" ANALYSE_USE_TIME_ARR = [] #模拟记录每次的处理时间(传入1:表示有处理,传入3:表示没处理)
#!/usr/bin/python import json import settings import pymongo from bson import ObjectId from Logger import Logger from database import MongoDB DATABASE = settings.database COLLECTION = settings.collection # Create an instance of logger class logger = Logger() class Media(dict): """ A Media model that creates movie/show type document on mongodb """ def __init__(self, dict): """ Class instructor """ pass def store(self): """ A function in this class to store a media, a movie or a show into the database """ client = MongoDB.setupConnection() try: db = client[DATABASE] collection = db[COLLECTION] return collection.insert_one(self).inserted_id except Exception as error:
class PullFTP(object): def __init__(self, source, logger=None, sleeping=False) : # General Attributes self.source = source self.sleeping = sleeping self.connected = False self.ftp = None self.sftp = None self.file = None self.ls = {} self.lsold = {} self.lspath = '' self.pulllst = [] self.originalDir = '' self.destDir = '' if logger is None: self.logger = Logger(PXPaths.LOG + 'rx_' + source.name + '.log', 'INFO', 'RX' + name) # Enable logging self.logger = self.logger.getLogger() else: self.logger = logger if self.source.delete and self.sleeping : return if self.source.protocol == 'ftp': self.ftp = self.ftpConnect() if self.ftp != None : self.chdir = self.ftp.cwd self.delete = self.ftp.delete self.fget = self.ftp_get self.lsdir = self.ftp_do_ls self.quit = self.ftp.quit if self.source.protocol == 'sftp': self.sftp = self.sftpConnect() if self.sftp != None : self.chdir = self.sftp.chdir self.delete = self.sftp.remove self.fget = self.sftp.get self.lsdir = self.sftp_do_ls self.quit = self.sftp_quit # callback to strip and save the output from ls of directory def callback_line(self,block): # strip ls line to its most important info fil,desc = self.ls_line_stripper(block) # keep only if we are interested in that file for filepattern in self.pulllst : self.logger.debug("fil = (%s)" % fil) if re.compile(filepattern).match(fil): self.logger.debug("fil = OK") self.ls[fil] = desc # going to a certain directory def cd(self, path): timex = AlarmFTP('FTP change working directory timeout') try : # gives 10 seconds to get there timex.alarm(10) self.chdir(self.originalDir) self.chdir(path) timex.cancel() return True except : timex.cancel() (type, value, tb) = sys.exc_info() self.logger.warning("Could not cd to directory %s" % path ) self.logger.warning(" Type: %s, Value: %s" % (type ,value)) return False # close connection... def close(self): self.connected = False # connection did not work if self.ftp == None and self.sftp == None : return timex = AlarmFTP(self.source.protocol + ' connection timeout') try : # gives 10 seconds to close the connection timex.alarm(10) self.quit() timex.cancel() except : timex.cancel() (type, value, tb) = sys.exc_info() self.logger.warning("Could not close connection") self.logger.warning(" Type: %s, Value: %s" % (type ,value)) # find differences between current ls and last ls # only the newer or modified files will be kept... def differ(self): # get new list and description new_lst = self.ls.keys() new_desc = self.ls new_lst.sort() # get old list and description self.load_ls_file(self.lspath) old_lst = self.lsold.keys() old_desc = self.lsold old_lst.sort() # compare filelst = [] desclst = {} for f in new_lst : # keep a newer entry if not f in old_lst : filelst.append(f) desclst[f] = new_desc[f] continue # keep a modified entry if new_desc[f] != old_desc[f] : filelst.append(f) desclst[f] = new_desc[f] continue return filelst,desclst # check for pattern matching in directory name def dirPattern(self,path) : """ Replace pattern in directory... """ ndestDir = '' DD = path.split("/") for ddword in DD[1:] : ndestDir += '/' if ddword == "" : continue nddword = "" DW = ddword.split("$") for dwword in DW : nddword += self.matchPattern(dwword,dwword) ndestDir += nddword return ndestDir # open connection... ftp def ftpConnect(self): timex = AlarmFTP('FTP connection timeout') try: # gives 30 seconds to open the connection timex.alarm(30) ftp = ftplib.FTP(self.source.host, self.source.user, self.source.passwd) if self.source.ftp_mode == 'active': ftp.set_pasv(False) else: ftp.set_pasv(True) try : self.originalDir = ftp.pwd() except: (type, value, tb) = sys.exc_info() self.logger.warning("Unable to ftp.pwd (Type: %s, Value: %s)" % (type ,value)) timex.cancel() self.connected = True return ftp except FtpTimeoutException : timex.cancel() self.logger.warning("FTP connection timed out after 30 seconds..." ) except: timex.cancel() (type, value, tb) = sys.exc_info() self.logger.error("Unable to connect to %s (user:%s). Type: %s, Value: %s" % (self.source.host, self.source.user, type ,value)) # ftp do an ls in the current directory, write in file path def ftp_do_ls(self): self.ls = {} timex = AlarmFTP('FTP retrieving list') timex.alarm(self.source.timeout_get) try : ls=self.ftp.retrlines('LIST',self.callback_line ) timex.cancel() return True except FtpTimeoutException : timex.cancel() self.logger.warning("FTP doing ls timed out after %s seconds..." % self.source.timeout_get ) except : timex.cancel() (type, value, tb) = sys.exc_info() self.logger.warning("Could not ls directory %s" % self.destDir) self.logger.warning(" Type: %s, Value: %s" % (type ,value)) return False # ftp retrieve a file def ftp_get(self, remote_file, local_file): file=open(local_file,'wb') self.ftp.retrbinary('RETR ' + remote_file, file.write ) file.close() # pulling files and returning the list def get(self): # if pull is sleeping and we delete files... nothing to do # if we don't delete files, we will keep the directory state if self.source.delete and self.sleeping : return # log that we are waking up if not self.sleeping : self.logger.info("pull %s is waking up" % self.source.name ) # getting our alarm ready timex = AlarmFTP('FTP timeout') # files list to return files_pulled = [] # connection did not work if self.ftp == None and self.sftp == None : return files_pulled # loop on all directories where there are pulls to do for lst in self.source.pulls : self.destDir = lst[0] self.pulllst = lst[1:] pdir = self.dirPattern(self.destDir) if pdir != '' : self.destDir = pdir self.destDir = self.destDir[1:] # cd to that directory self.logger.debug(" cd %s" % self.destDir) ok = self.cd(self.destDir) if not ok : continue # create ls filename for that directory pdir = lst[0] pdir = pdir.replace('${','') pdir = pdir.replace('}','') self.lspath = PXPaths.RXQ + self.source.name + '/.ls' + pdir.replace('/','_') # ls that directory ok = self.lsdir() if not ok : continue # if we are sleeping and we are here it is because # this pull is retrieving difference between directory content # so write the directory content without retrieving files if self.sleeping : ok = self.write_ls_file(self.lspath) continue # get the file list from the ls filelst = self.ls.keys() desclst = self.ls # if we dont delete, get file list from difference in ls if not self.source.delete : filelst,desclst = self.differ() if len(filelst) == 0 : ok = self.write_ls_file(self.lspath) continue # get remote file time for all files timelst,filelst,desclst = self.remote_time_sort(filelst,desclst) # retrieve the files files_notretrieved = [] for idx,remote_file in enumerate(filelst) : timex.alarm(self.source.timeout_get) local_file = self.local_filename(remote_file,desclst,timelst) try : ok = self.retrieve(remote_file, local_file) if ok : if self.source.delete : self.rm(remote_file) files_pulled.append(local_file) # setting access,modify time to remote time ftime = time.mktime(timelst[remote_file]) os.utime(local_file,(ftime,ftime) ) else : files_notretrieved.extend(filelst[idx:]) self.logger.warning("problem when retrieving %s " % remote_file ) break # if batch is reached stop there if len(files_pulled) == self.source.batch : if idx != len(filelst)-1 : files_notretrieved.extend(filelst[idx+1:]) break timex.cancel() except FtpTimeoutException : timex.cancel() files_notretrieved.extend(filelst[idx:]) self.logger.warning("FTP timed out retrieving %s " % remote_file ) break except : timex.cancel() files_notretrieved.extend(filelst[idx:]) (type, value, tb) = sys.exc_info() self.logger.error("Unable write remote file %s in local file %s" % (remote_file,local_file)) self.logger.error(" Type: %s, Value: %s" % (type ,value)) break # files not retrieved are removed from the file list # this allow pull to recover from error on next pass for f in files_notretrieved : del self.ls[f] # save ls file ok = self.write_ls_file(self.lspath) # if reached the batch limit : break here if len(files_pulled) == self.source.batch : self.logger.warning("break retrieving... batch reached") break # if we had a problem break here if len(files_notretrieved) > 0 : self.logger.warning("break retrieving... because of error") break return files_pulled # parse a file containing an ls and return the file entries def load_ls_file(self,path): self.lsold = {} if not os.path.isfile(path) : return True try : # open/read.. file=open(path,'rb') lines=file.readlines() file.close() # get filenames for line in lines : parts = line.split() fil = parts[-1] self.lsold[fil] = line return True except: self.logger.error("Unable to parse files from %s" % path ) return False # create local filename def local_filename(self,filename,desclst,timelst): # create local filename with date/time on host... YYYYMMDDHHMM_filename if self.source.pull_prefix == 'HDATETIME' : ftime = timelst[filename] datetimestr = time.strftime("%Y%m%d%H%M",ftime) local_file = PXPaths.RXQ + self.source.name + '/' + datetimestr + '_' + filename return local_file # create local filename with config defined prefix ... prefix_filename if self.source.pull_prefix != '' : local_file = PXPaths.RXQ + self.source.name + '/' + self.source.prefix + '_' + filename return local_file # create local filename with same name as remote file (default) local_file = PXPaths.RXQ + self.source.name + '/' + filename return local_file # ls line stripper def ls_line_stripper(self,iline): oline = iline oline = oline.strip() oline = oline.replace('\t',' ') opart1 = oline.split(' ') opart2 = [] for p in opart1 : if p == '' : continue opart2.append(p) fil = opart2[-1] desc = opart2[0] + ' ' + ' '.join(opart2[-5:]) + '\n' return fil,desc # replace a matching pattern by its value in the directory name def matchPattern(self,keywd,defval) : """ Matching keyword with different patterns """ if keywd[:6] == "{YYYY}" : return time.strftime("%Y", time.gmtime()) + keywd[6:] if keywd[:9] == "{YYYY-1D}" : epoch = time.mktime(time.gmtime()) - 24*60*60 return time.strftime("%Y", time.localtime(epoch) ) + keywd[9:] if keywd[:4] == "{MM}" : return time.strftime("%m", time.gmtime()) + keywd[4:] if keywd[:7] == "{MM-1D}" : epoch = time.mktime(time.gmtime()) - 24*60*60 return time.strftime("%m", time.localtime(epoch) ) + keywd[7:] if keywd[:5] == "{JJJ}" : return time.strftime("%j", time.gmtime()) + keywd[5:] if keywd[:8] == "{JJJ-1D}" : epoch = time.mktime(time.gmtime()) - 24*60*60 return time.strftime("%j", time.localtime(epoch) ) + keywd[8:] if keywd[:10] == "{YYYYMMDD}" : return time.strftime("%Y%m%d", time.gmtime()) + keywd[10:] if keywd[:13] == "{YYYYMMDD-1D}" : epoch = time.mktime(time.gmtime()) - 24*60*60 return time.strftime("%Y%m%d", time.localtime(epoch) ) + keywd[13:] if keywd[:13] == "{YYYYMMDD-2D}" : epoch = time.mktime(time.gmtime()) - 48*60*60 return time.strftime("%Y%m%d", time.localtime(epoch) ) + keywd[13:] if keywd[:13] == "{YYYYMMDD-3D}" : epoch = time.mktime(time.gmtime()) - 72*60*60 return time.strftime("%Y%m%d", time.localtime(epoch) ) + keywd[13:] if keywd[:13] == "{YYYYMMDD-4D}" : epoch = time.mktime(time.gmtime()) - 96*60*60 return time.strftime("%Y%m%d", time.localtime(epoch) ) + keywd[13:] if keywd[:13] == "{YYYYMMDD-5D}" : epoch = time.mktime(time.gmtime()) - 120*60*60 return time.strftime("%Y%m%d", time.localtime(epoch) ) + keywd[13:] return defval # get remote file time def remote_file_time(self,filename,desc): ftime = time.localtime() try : line = desc[filename] line = line.strip() parts = line.split() datestr=' '.join(parts[-4:-1]) fmt4 = "%b" fmt3 = "%d" fmt2 = "%Y" if len(parts[-2]) == 5 : fmt2 = "%H:%M %Y" Y = time.strftime("%Y",time.gmtime()) datestr = datestr + ' ' + Y if parts[-3][0].isalpha() : fmt3 = "%b" if parts[-4][0].isdigit() : fmt4 = "%d" format = fmt4 + " " + fmt3 + " " + fmt2 ftime = time.strptime(datestr,format) except : pass return ftime # remote time sort of file to pull def remote_time_sort(self,ifilelst,idesclst): timelst = [ self.remote_file_time(rfile,idesclst) for idx,rfile in enumerate(ifilelst) ] templst = [ (time.mktime(timelst[idx]),timelst[idx],rfile,idesclst[rfile]) for idx,rfile in enumerate(ifilelst) ] templst.sort() filelst = [] desclst = {} timelst = {} for tup in templst : epoch, rtime, rfile, rdesc = tup filelst.append(rfile) desclst[rfile] = rdesc timelst[rfile] = rtime return timelst,filelst,desclst # retrieve a file def retrieve(self, remote_file, local_file): try : self.fget( remote_file, local_file) return True except : os.unlink(local_file) (type, value, tb) = sys.exc_info() self.logger.error("Unable write remote file %s in local file %s. Type: %s, Value: %s" % \ (remote_file,local_file,type,value)) return False # some systems do not permit deletion... so pass exception on that def rm(self, path): try : self.delete(path) except : (type, value, tb) = sys.exc_info() self.logger.warning("Could not delete %s" % path ) self.logger.warning("Type: %s, Value: %s" % (type ,value)) # open connection... sftp def sftpConnect(self): timex = AlarmFTP('SFTP connection timeout') # gives 30 seconds to open the connection try: timex.alarm(30) self.t = None if self.source.port == None : self.t = paramiko.Transport(self.source.host) else: t_args = (self.source.host,self.source.port) self.t = paramiko.Transport(t_args) if self.source.ssh_keyfile != None : #TODO, implement password to use to decrypt the key file, if it's encrypted key=DSSKey.from_private_key_file(self.source.ssh_keyfile,password=None) self.t.connect(username=self.source.user,pkey=key) else: self.t.connect(username=self.source.user,password=self.source.passwd) self.sftp = paramiko.SFTP.from_transport(self.t) # WORKAROUND without going to '.' originalDir was None self.sftp.chdir('.') self.originalDir = self.sftp.getcwd() timex.cancel() self.connected = True return self.sftp except FtpTimeoutException : timex.cancel() try : self.sftp.close() except : pass try : self.t.close() except : pass self.logger.error("SFTP connection timed out after %d seconds... retrying" % 30 ) except: timex.cancel() (type, value, tb) = sys.exc_info() self.logger.error("Unable to connect to %s (user:%s). Type: %s, Value: %s" % (self.source.host, self.source.user, type ,value)) try : self.sftp.close() except : pass try : self.t.close() except : pass # do an ls in the current directory, write in file path def sftp_do_ls(self): self.ls = {} timex = AlarmFTP('SFTP retrieving list') timex.alarm(self.source.timeout_get) try : dir_attr = self.sftp.listdir_attr() for index in range(len(dir_attr)): attr = dir_attr[index] line = attr.__str__() self.callback_line(line) timex.cancel() return True except FtpTimeoutException : timex.cancel() self.logger.warning("SFTP doing ls timed out after %s seconds..." % self.source.timeout_get ) except : timex.cancel() (type, value, tb) = sys.exc_info() self.logger.warning("Could not ls directory %s" % self.destDir) self.logger.warning(" Type: %s, Value: %s" % (type ,value)) return False # sftp quit = close connection... def sftp_quit(self): self.sftp.close() self.t.close() # write ls file def write_ls_file(self,path): filelst = self.ls.keys() desclst = self.ls filelst.sort() try : # open/write.. file=open(path,'wb') for f in filelst : file.write(desclst[f]) file.close() return True except: self.logger.error("Unable to write ls to file %s" % path ) return False
def prt(self): Logger.dbg('x : {0} - y : {1}'.format(self.x, self.y))
dryRun = False try: if sys.argv.index('--dryrun') > 0: dryRun = True except ValueError: pass def timestamp(): ts = time.time() return datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S') bot = Poloniex(config.get("API", "apikey"), config.get("API", "secret")) log = Logger() #total lended global variable totalLended = {} def refreshTotalLended(): global totalLended, rateLended cryptoLended = bot.returnActiveLoans() totalLended = {} rateLended = {} cryptoLendedSum = Decimal(0) cryptoLendedRate = Decimal(0) for item in cryptoLended["provided"]:
class Trainer(object): def __init__(self, cfg, ckpt_path=None): self.cfg = cfg random.seed(self.cfg.SEED) torch.manual_seed(self.cfg.SEED) torch.cuda.manual_seed(self.cfg.SEED) dt_now = dt.datetime.now().strftime("%Y-%m-%d %H:%M:%S") self.experiment_name = self.cfg.PROJECT_NAME + '_' + dt_now self.experiment_path = os.path.join(self.cfg.EXPERIMENTS_PATH, self.experiment_name) self.make_folders() self.logger = Logger(self.experiment_path, to_file=True, to_tensorboard=True) #self.env = AtariEnv(self.cfg) self.env = CartPoleEnv(self.cfg) if self.cfg.USE_GPU: self.gpu_id = 0 self.device = torch.device('cuda', self.gpu_id) else: self.device = torch.device('cpu') #self.target_model = DQN(self.cfg).to(self.device) self.target_model = DQNMLP(self.cfg).to(self.device) self.target_model.eval() #self.model = DQN(self.cfg).to(self.device) self.model = DQNMLP(self.cfg).to(self.device) self.model.train() self.logger.log_config(self.cfg) self.logger.log_pytorch_model(self.model) self.ckpt_path = os.path.join( self.experiment_path, 'ckpt', self.target_model.model_name + '.weights') if self.cfg.OPTIM == 'adam': self.optimizer = optim.Adam(filter(lambda p: p.requires_grad, self.model.parameters()), lr=self.cfg.LEARNING_RATE) #self.optimizer = optim.Adam(parameters, lr=self.cfg.LEARNING_RATE) elif self.cfg.OPTIM == 'rms-prop': self.optimizer = optim.RMSprop(filter(lambda p: p.requires_grad, self.model.parameters()), lr=self.cfg.LEARNING_RATE) #self.optimizer = optim.RMSprop(parameters, # lr=self.cfg.LEARNING_RATE) elif self.cfg.OPTIM == 'sgd': self.optimizer = optim.SGD(filter(lambda p: p.requires_grad, self.model.parameters()), lr=self.cfg.LEARNING_RATE) if ckpt_path is not None: self.model.load_state_dict(torch.load(ckpt_path)) def train(self): """ Procedure to run the training procedure """ self.target_model.load_state_dict(self.model.state_dict()) experience_replay = ExperienceReplay(self.cfg) epsilon = self.cfg.EPS_START drop_epsilon = (self.cfg.EPS_START - self.cfg.EPS_END) / self.cfg.EPS_DECAY global_step = 0 epsilon_step = 0 for e in range(self.cfg.MAX_EPISODES): self.env.reset() state = self.env.get_state() while not self.env.done: global_step += 1 if self.cfg.RENDER: self.env.render() if global_step > self.cfg.TRAIN_START: epsilon_step += 1 epsilon = self.cfg.EPS_END + (self.cfg.EPS_START - self.cfg.EPS_END) * \ math.exp(-1. * epsilon_step / self.cfg.EPS_DECAY) self.logger.log_value('epsilon', global_step, epsilon, print_value=False, to_file=False) if random.random() < epsilon: # epsilon-greedy exploration action = torch.tensor( [[random.randrange(self.cfg.NUM_ACTIONS)]], dtype=torch.long) else: with torch.no_grad(): q_pred = self.model(state.to(self.device)) _, action = q_pred.max(1) action = action.view(1, 1).cpu() reward = self.env.step(action.item()) r = np.clip(reward, -1.0, 1.0) r = torch.tensor([reward]) if not self.env.done: next_state = self.env.get_state() else: next_state = None experience_replay.push(state, action, next_state, r) #total_experience.add(total_experience.get_max(), (w_state, action, r, next_w_state, done)) state = next_state if global_step > self.cfg.TRAIN_START and len( experience_replay) > self.cfg.BATCH_SIZE: transitions = experience_replay.sample() batch = Transition(*zip(*transitions)) non_final_mask = torch.tensor(tuple( map(lambda s: s is not None, batch.next_state)), device=self.device, dtype=torch.uint8) non_final_next_states = torch.cat( [s for s in batch.next_state if s is not None]) state_batch = torch.cat(batch.state).to(self.device) action_batch = torch.cat(batch.action).to(self.device) reward_batch = torch.cat(batch.reward).to(self.device) # reward standardization if self.cfg.STD_REWARDS: reward_batch = (reward_batch - reward_batch.mean()) / ( reward_batch.std() + np.finfo(np.float32).eps.item()) state_action_values = self.model(state_batch).gather( 1, action_batch) next_state_values = torch.zeros(self.cfg.BATCH_SIZE, device=self.device) if self.cfg.DOUBLE_DQN: _, next_state_actions = self.model( non_final_next_states.to(self.device)).max( 1, keepdim=True) next_state_values[non_final_mask] = self.target_model( non_final_next_states.to(self.device)).gather( 1, next_state_actions).squeeze(1).detach() else: next_state_values[non_final_mask] = self.target_model( non_final_next_states.to( self.device)).max(1)[0].detach() expected_state_action_values = ( next_state_values * self.cfg.GAMMA) + reward_batch if self.cfg.HUBER_LOSS: loss = F.smooth_l1_loss( state_action_values, expected_state_action_values.unsqueeze(1)) else: loss = F.mse_loss( state_action_values, expected_state_action_values.unsqueeze(1)) # training op self.optimizer.zero_grad() loss.backward() for param in self.model.parameters(): param.grad.data.clamp_(-1, 1) self.optimizer.step() self.logger.log_value('loss', global_step, loss.item(), print_value=False, to_file=False) #for i in range(self.cfg.BATCH_SIZE): # total_experience.update(b_i[i], errors[i]) # update target weights if (global_step % self.cfg.TARGET_UPDATE) == 0: self.target_model.load_state_dict(self.model.state_dict()) self.logger.log_episode('DQN agent', e, self.env.total_reward) if (e % self.cfg.SAVE_STEP) == 0: torch.save(self.target_model.state_dict(), self.ckpt_path) self.logger.log_variables() torch.save(self.target_model.state_dict(), self.ckpt_path) self.logger.log_variables() def make_folders(self): """ Creates folders for the experiment logs and ckpt """ os.makedirs(os.path.join(self.experiment_path, 'ckpt')) os.makedirs(os.path.join(self.experiment_path, 'logs'))
""" instantiate PiPAD + RoPi sensors """ pipad_led = PiPadLED() pipad_buttons = PiPadButtons() pipad_stick = PiPadStick() pipad_rotary = PiPadRotary() ropi_led = RoPiLED(_ropi['remote_address']) ropi_buzzer = RoPiBuzzer(_ropi['remote_address']) ropi_servo = RoPiServo(_ropi['remote_address']) ropi_track = RoPiTrack(_ropi['remote_address']) ropi_distance = RoPiDistance(_ropi['remote_address']) #ropi_screen = RoPiScreen(_ropi['remote_address']) ropi_motor = RoPiMotor(_ropi['remote_address']) # LOGGING """ instantiate logger """ log = Logger(_pipad['DEVEL_LOG']) # GLOBALS """ global variables """ rotary_counter_lr = 0 rotary_current_counter_lr = 0 rotary_counter_ud = 0 rotary_current_counter_ud = 0 rotary_reverse = 0 stick_autonomous = False stick_position = 0 # COLOR A LED """ color a specified RGBLEB *to_color* *for_time* """
post_list.append(tweet) logger.info("{0} - {1} : {2}".format( id, screen_name, text)) ignore_list.append(id) logger.info("Got {0} results".format(c)) except tweepy.TweepError as e: logger.exception( "Could not connect to Twitter - are your credentials correct?") if __name__ == '__main__': Config.load('config.json') logger = Logger('twitter-contestant') # Intilialize Tweepy auth = tweepy.OAuthHandler(Config.consumer_key, Config.consumer_secret) auth.set_access_token(Config.access_token_key, Config.access_token_secret) api = tweepy.API(auth, wait_on_rate_limit=True, wait_on_rate_limit_notify=True) # Initialize ignorelist ignore_list = IgnoreList("ignorelist") # Initialize scheduler scheduler = BlockingScheduler() # First run
class Driver: def __init__(self): self.state = State(0) self.params = Params() self.status = Status(self.params) self.sleep_time = 1 self.pwm_read = PwmRead(self.params.pin_mode_in, self.params.pin_servo_in, self.params.pin_thruster_in) self.pwm_out = PwmOut(self.params.pin_servo_out, self.params.pin_thruster_out) self.pid = PositionalPID() self.logger = Logger() self.logger.open() def load(self, filename): print('loading', filename) f = open(filename, "r") line = f.readline() line = f.readline() self.state.time_limit = int(line.split()[1]) # Time Limit line = f.readline() self.sleep_time = float(line.split()[1]) # Sleep time line = f.readline() line = f.readline() line = f.readline() p = float(line.split()[1]) # P line = f.readline() i = float(line.split()[1]) # I line = f.readline() d = float(line.split()[1]) # D self.pid.setPID(p, i, d) line = f.readline() line = f.readline() line = f.readline() num = int(line.split()[1]) # Number of waypoints line = f.readline() for i in range(num): line = f.readline() self.status.waypoint.addPoint(float(line.split()[0]), float(line.split()[1])) f.close() return def doOperation(self): while self.state.inTimeLimit(): self.readPWM() self.readGps() mode = self.getMode() if mode == 'RC': self.remoteControl() elif mode == 'AN': self.autoNavigation() self.outPWM() self.printLog() time.sleep(self.sleep_time) return def getMode(self): return self.status.mode def updateMode(self): mode_duty_ratio = self.pwm_read.pulse_width[0] if mode_duty_ratio < 1500: self.status.mode = 'RC' elif mode_duty_ratio >= 1500: self.status.mode = 'AN' return def readGps(self): self.status.readGps() self.updateMode() #if self.status.isGpsError(): #self.status.mode = 'RC' return def updateStatus(self): status = self.status status.calcTargetDirection() status.calcTargetDistance() status.updateTarget() return def readPWM(self): self.pwm_read.measurePulseWidth() self.pwm_out.servo_pulsewidth = self.pwm_read.pulse_width[1] self.pwm_out.thruster_pulsewidth = self.pwm_read.pulse_width[2] return def outPWM(self): self.pwm_out.updatePulsewidth() return def autoNavigation(self): self.updateStatus() boat_direction = self.status.boat_direction target_direction = self.status.target_direction servo_pulsewidth = self.pid.getStepSignal(target_direction, boat_direction) self.pwm_out.servo_pulsewidth = servo_pulsewidth self.pwm_out.thruster_pulsewidth = 1880 return def remoteControl(self): # Do nothing return def printLog(self): timestamp_string = self.status.timestamp_string mode = self.getMode() latitude = self.status.latitude longitude = self.status.longitude speed = self.status.speed direction = self.status.boat_direction servo_pw = self.pwm_out.servo_pulsewidth thruster_pw = self.pwm_out.thruster_pulsewidth t_direction = self.status.target_direction t_distance = self.status.target_distance target = self.status.waypoint.getPoint() t_latitude = target[0] t_longitude = target[1] print(timestamp_string) print( '[%s MODE] LAT=%.7f, LON=%.7f, SPEED=%.2f [km/h], DIRECTION=%lf' % (mode, latitude, longitude, speed, direction)) print('DUTY (SERVO, THRUSTER): (%lf, %lf) [us]' % (servo_pw, thruster_pw)) print('TARGET (LATITUDE, LONGITUDE): (%lf, %lf)' % (t_latitude, t_longitude)) print('TARGET (DIRECTION, DISTANCE): (%lf, %lf [m])' % (t_direction, t_distance)) print('') log_list = [ timestamp_string, mode, latitude, longitude, direction, ] self.logger.write(log_list) return def finalize(self): self.logger.close() self.pwm_out.finalize() return
def index(): logger = Logger.getInstance() user_ip = request.remote_addr logger.log("index", user_ip) return render_template("index.html")
def main(): usage = "Usage: %prog [options] <package name>" parser = ArgumentParser(usage) parser.add_argument("-s", "--spec-path", dest="specPath", default="../../SPECS") parser.add_argument("-l", "--log-path", dest="logPath", default="../../stage/LOGS") parser.add_argument("-a", "--source-rpm-path", dest="sourceRpmPath", default="../../stage/SRPMS") parser.add_argument("-j", "--output-dir", dest="outputDirPath", default="../../stage/") parser.add_argument("-c", "--pullsources-config", dest="pullsourcesConfig", default="pullsources.conf") parser.add_argument("-f", "--pkg-blacklist-file", dest="pkgBlacklistFile", default=None) parser.add_argument("-p", "--generate-pkg-list", dest="generatePkgList", default=False, action="store_true") parser.add_argument("-y", "--generate-yaml-files", dest="generateYamlFiles", default=False, action="store_true") options = parser.parse_args() errorFlag = False cmdUtils = CommandUtils() try: if not os.path.isdir(options.logPath): cmdUtils.runCommandInShell("mkdir -p " + options.logPath) logger = Logger.getLogger(options.logPath + "/generateYamlFiles") if options.generateYamlFiles: if (options.pkgBlacklistFile is not None and options.pkgBlacklistFile != "" and not os.path.isfile(options.pkgBlacklistFile)): logger.error("Given package blacklist file is not valid:" + options.pkgBlacklistFile) errorFlag = True if not os.path.isdir(options.specPath): logger.error("Given Specs Path is not a directory:" + options.specPath) errorFlag = True if not os.path.isdir(options.sourceRpmPath): logger.error("Given SRPM Path is not a directory:" + options.sourceRpmPath) errorFlag = True if options.generateYamlFiles and not os.path.isfile( options.pullsourcesConfig): logger.error("Given Source config file is not a valid file:" + options.pullsourcesConfig) errorFlag = True if errorFlag: logger.error( "Found some errors. Please fix input options and re-run it.") sys.exit(1) if options.generateYamlFiles: if not os.path.isdir(options.outputDirPath): cmdUtils.runCommandInShell("mkdir -p " + options.outputDirPath) constants.setSpecPath(options.specPath) constants.setSourceRpmPath(options.sourceRpmPath) constants.setLogPath(options.logPath) constants.setPullSourcesConfig(options.pullsourcesConfig) constants.initialize() # parse SPECS folder SPECS() if options.generatePkgList: buildPackagesList(options.outputDirPath + "/packages_list.csv") elif options.generateYamlFiles: blackListPkgs = readBlackListPackages(options.pkgBlacklistFile) buildSourcesList(options.outputDirPath, blackListPkgs, logger) buildSRPMList(options.sourceRpmPath, options.outputDirPath, blackListPkgs, logger) except Exception as e: print("Caught Exception: " + str(e)) traceback.print_exc() sys.exit(1) sys.exit(0)
class DrebinUtil: datasetPath = "" #PATH DEL DATASET testPath = "" #PATH DOVE INSERIRE GLI APK OFFUSCATI dictionary = "" #PATH DEL DICTIONARY logLevel = 3 __logger = "" ##The constructor: # @param datasetPath = path of the dataset; # @param testPath = path of the workspace; # @param dictionaryPath = path of the drebin dictionary; def __init__(self, datasetPath, testPath, dictionaryPath): self.datasetPath = datasetPath self.testPath = testPath self.dictionary = dictionaryPath self.logger = Logger(self.logLevel) ##This method read from the dictionary the family of a sample; # @param fileName = the name of the sample to find the family ; def readMalwareFamily(self, fileName): bashArray = ["grep", fileName, self.dictionary] process = subprocess.Popen(bashArray, stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, error = process.communicate() output = str(output) if (error or not output): malwareFamily = "UKNOWN" else: malwareFamilyList = output.split(",") malwareFamily = malwareFamilyList[1] malwareFamily = malwareFamily.replace("\r", " ") malwareFamily = malwareFamily.replace("\n", " ") self.logger.log("INFO", "MALWARE FAMILY: " + malwareFamily) return (malwareFamily) ##This method change the name of a file (used to add .apk extension); # @param fileName = the original filename; # @param newName = the new name; def prepareFile(self, fileName, newName): bashCommand = "mv " + self.datasetPath + fileName + " " + self.testPath + newName process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE, stderr=subprocess.PIPE) output, error = process.communicate() if (error): self.logger.log("ERROR", "ERROR PREPARING APK FILE") return (-1) else: self.logger.log("INFO", "ADD APK EXTENSION: " + newName) return (0) def cleanBadApk(self, newName, mapName, obfuscatedName, packageName): bashCommand = "rm " + self.testPath + newName try: subprocess.check_output(bashCommand, stderr=subprocess.STDOUT, shell=True) self.logger.log("INFO", "REAL APK REMOVED") except subprocess.CalledProcessError: self.logger.log("ERROR", "ERROR REMOVING REAL APK") bashCommand = "rm " + self.testPath + mapName try: subprocess.check_output(bashCommand, stderr=subprocess.STDOUT, shell=True) self.logger.log("INFO", "MAP FILE REMOVED") except subprocess.CalledProcessError: self.logger.log("ERROR", "ERROR REMOVING MAP FILE") bashCommand = "rm " + self.testPath + obfuscatedName try: subprocess.check_output(bashCommand, stderr=subprocess.STDOUT, shell=True) self.logger.log("INFO", "OBFUSCATED APK REMOVED") except subprocess.CalledProcessError: self.logger.log("ERROR", "ERROR REMOVING OBFUSCATED APK") def moveGoodApk(self, newName, mapName, obfuscatedName, packageName, goodAPKPath): self.logger.log("INFO", "THIS APK IS GOOD, MOVING TO NEW LOCATION") folder = goodAPKPath + "/" + packageName bashCommand = "mkdir " + folder try: subprocess.check_output(bashCommand, stderr=subprocess.STDOUT, shell=True) self.logger.log("INFO", "CREATED DIRECTORY: " + folder) except subprocess.CalledProcessError: self.logger.log("ERROR", "ERROR CREATING DIRECTORY: " + folder) bashCommand = "mv " + self.testPath + newName + " " + folder + "/" + newName try: subprocess.check_output(bashCommand, stderr=subprocess.STDOUT, shell=True) self.logger.log("INFO", "REAL APK MOVED") except subprocess.CalledProcessError: self.logger.log("ERROR", "ERROR MOVING REAL APK") bashCommand = "mv " + self.testPath + mapName + " " + folder + "/" + mapName try: subprocess.check_output(bashCommand, stderr=subprocess.STDOUT, shell=True) self.logger.log("INFO", "MAP FILE MOVED") except subprocess.CalledProcessError: self.logger.log("ERROR", "ERROR MOVING MAP FILE") bashCommand = "mv " + self.testPath + obfuscatedName + " " + folder + "/" + obfuscatedName try: subprocess.check_output(bashCommand, stderr=subprocess.STDOUT, shell=True) self.logger.log("INFO", "OBFUSCATED APK MOVED") except subprocess.CalledProcessError: self.logger.log("ERROR", "ERROR MOVING OBFUSCATED APK")
class InfraredSensor: DR = 16 DL = 19 interrupted = False thread = None on = False left = 1 right = 1 def __init__(self, client, service_name, debug=False): self.client = client self.serviceName = service_name self.logger = Logger("InfraredSensor", debug) self.source = Observable.interval(50).map(lambda i: self.get_state()) GPIO.setup(self.DR, GPIO.IN, GPIO.PUD_UP) GPIO.setup(self.DL, GPIO.IN, GPIO.PUD_UP) self.client.publish(self.serviceName + "/state/ir-sensor/state", "OFF", 1, True) self.client.publish(self.serviceName + "/state/ir-sensor/state/left", "OFF", 1, True) self.client.publish(self.serviceName + "/state/ir-sensor/state/right", "OFF", 1, True) self.client.message_callback_add( self.serviceName + "/control/ir-sensor/#", self.on_message) def on_message(self, client, userdata, msg): self.logger.info(msg.topic + ": " + msg.payload) try: path = msg.topic.split("/") if len(path) > 1 and path[0] == self.serviceName and path[ 1] == "control": # mutinus/control/# if len(path) > 2 and path[ 2] == "ir-sensor": # mutinus/control/ir-sensor self.logger.info(msg.payload) if msg.payload == "ON": self.start() else: self.stop() except: self.logger.error("Unexpected Error!") traceback.print_exc() def get_state(self): left = GPIO.input(self.DL) right = GPIO.input(self.DR) return [left, right] def subscribe(self, on_next): return self.source.subscribe( on_next=on_next, on_error=lambda e: self.logger.error(str(e)), on_completed=lambda: self.logger.info("Subscription completed")) def looper(self): while not self.interrupted: try: state = self.get_state() self.logger.debug("Left: " + str(state[0]) + ", Right: " + str(state[1])) if state[0] != self.left: self.left = state[0] payload = "ON" if self.left == 0 else "OFF" self.client.publish( self.serviceName + "/state/ir-sensor/left", payload, 0, False) if state[1] != self.right: self.right = state[1] payload = "ON" if self.right == 0 else "OFF" self.client.publish( self.serviceName + "/state/ir-sensor/right", payload, 0, False) time.sleep(0.05) except: self.logger.error("Unexpected Error!") traceback.print_exc() self.logger.info("Exiting looper") def start(self): if self.thread is not None: self.client.publish(self.serviceName + "/state/ir-sensor", "ON", 1, True) self.interrupted = False self.thread = threading.Thread(target=self.looper) self.thread.start() return self.thread def stop(self): self.interrupted = True if self.thread is not None: self.client.publish(self.serviceName + "/state/ir-sensor", "OFF", 1, True) self.thread.join(5) self.thread = None
# -*- coding:utf-8 -*- # @Time: 2020/7/11 10:31 PM # @Author: dyf-2316 # @FileName: getData.py # @Software: PyCharm # @Project: Comment_Sentiment_Analysis # @Description: get data from html/json import re from Logger import Logger from config import * from WebCrawler.getPageSource import get_page_html, get_json_data mylogger = Logger("gatData").logger def get_search_url(keywords): """ 通过搜索界面URL与关键字的拼接,获取一个关键字商品搜索页面的URL。 :param keywords: (str) 想要查询商品的关键字 :return: (str) 含有关键字的商品搜索界面的URL """ search_url = SEARCH_URL.format(keywords) # logger传入message必须是字符串(用format或%拼接好),与print相区别 mylogger.debug("获取搜索界面URL -- {}".format(search_url)) return search_url product_already_get = ['100002076057', '47775809660', '51961736081', '51961736083', '51961736082',
def A1(F,G): P = IMPLICATION(F,IMPLICATION(G,F)) Logger(P,'A1 for '+F+' and '+G) return P