def run(self, parsedCommandLine): (buildSteps, self._custom_args) = (parsedCommandLine[0], parsedCommandLine[1]) # this build MUST have a project name to run if self._project_name == "": Utilities.failExecution("Project name not set") # if the user has not specified any build steps, run the default if len(buildSteps) == 0: buildSteps = self._build_steps # run the build for the user specified configuration else run for # all configurations (the user can restrict this to build for # debug or release versions) if "configuration" in self._custom_args: self._config = self._custom_args["configuration"] if self._config != "release" and self._config != "debug": Utilities.failExecution("Unknown configuration [%s]" % self._config) print("\nbuilding configuration [%s]\n" % self._config) self.executeBuildSteps(buildSteps) else: for configuration in self._configurations: print("\nbuilding configuration [%s]\n" % configuration) self._config = configuration self.executeBuildSteps(buildSteps) print("********************") print("* COMPLETE *") print("********************")
def main(): # training parameter result_path = 'results/housingLiR_1.mse' model_name = 'housing_shiftAndScale' # normalization = Preprocess.zero_mean_unit_var normalization = Preprocess.shift_and_scale # cols_not_norm = (0,7,12) cols_not_norm = [] # laod and preprocess training data training_data = loader.load_dataset('data/housing_train.txt') testing_data = loader.load_dataset('data/housing_test.txt') Preprocess.normalize_features_all(normalization, training_data[0], testing_data[0], cols_not_norm) # start training model = rm.LinearRegression() model.build(training_data[0], training_data[1]) training_mse = model.test(training_data[0], training_data[1], util.mse) testing_mse = model.test(testing_data[0], testing_data[1], util.mse) print 'Error for training data is:' print training_mse print 'Error for testing data is:' print testing_mse result = {} result['TrainingMSE'] = str(training_mse) result['TestingMSE'] = str(testing_mse) result['Theta'] = str(model.theta) # log the training result to file util.write_result_to_file(result_path, model_name, result)
def __init__(self, fitness=2): screen = pygame.display.get_surface() self.area = screen.get_rect() self.fitness = fitness self.awareness = random.randint(50,60) + self.fitness self.speed = random.randint(30,40) + self.fitness self.radius = self.awareness self.target = None self.tempTarget = (random.randint(self.area.left + 50, self.area.right - 50),random.randint(self.area.top + 50, self.area.bottom - 50)) self.high = False self.med = False self.low = False self.highFitness = Utilities.load_image('infectedSuper.png') self.medFitness = Utilities.load_image('infected1.png') self.lowFitness = Utilities.load_image('infected2.png') if(self.fitness >= 10): self.image, self.rect = self.highFitness elif(self.fitness >= 5): self.image, self.rect = self.medFitness elif(self.fitness < 5): self.image, self.rect = self.lowFitness self.low = True self.rect.center = (random.randint(self.area.left+6, self.area.right - 6), random.randint(self.area.top + 32, self.area.bottom - 6)) pygame.sprite.Sprite.__init__(self) self.active = True
def parseDependencyFile(self): dependencyFilePath = os.path.join(FileSystem.getDirectory(FileSystem.DEPENDENCIES), "dependencies.txt") if not os.path.exists(dependencyFilePath): Utilities.failExecution("dependency file [%s] does not exist" % dependencyFilePath) requiredProjects = [] with open(dependencyFilePath, 'r') as file: flag = False lineNum = 0 splitLine = None for line in file: splitLine = line.strip().split(None) if len(splitLine) == 0 or splitLine[0] == '#': continue if splitLine[0] == '-' + self._project_name: flag = True elif flag and '-' not in splitLine[0]: requiredProjects.append(splitLine) elif flag and '-' in splitLine[0] and ('-' + self._project_name) != splitLine[0]: flag = False elif not flag: continue else: Utilities.failExecution("Parse error in dependency file [%s] at line [%s]" % (dependencyFilePath, lineNum)) lineNum += 1 print("Required projects for project [%s] are %s" % (self._project_name, requiredProjects)) return requiredProjects
def __init__(self, vcs, parent = None): """ Constructor @param vcs reference to the version control object @param parent parent widget (QWidget) """ QDialog.__init__(self, parent) self.setupUi(self) self.vcsDirectoryCompleter = E4DirCompleter(self.vcsUrlEdit) self.vcsProjectDirCompleter = E4DirCompleter(self.vcsProjectDirEdit) self.protocolCombo.addItems(ConfigSvnProtocols) hd = Utilities.toNativeSeparators(QDir.homePath()) hd = os.path.join(unicode(hd), 'subversionroot') self.vcsUrlEdit.setText(hd) self.vcs = vcs self.localPath = unicode(hd) self.networkPath = "localhost/" self.localProtocol = True self.vcsProjectDirEdit.setText(Utilities.toNativeSeparators( Preferences.getMultiProject("Workspace") or Utilities.getHomeDir()))
def main(filename: 'str') -> None: ''' Finds the palindromes in the file and prints the number of occurences of them. ''' words = Utilities.tokenizeFile(filename) frequencies = computePalindromeFrequencies(words) Utilities.printFrequencies(frequencies)
def __init__(self, project, parent=None): """ Constructor @param project reference to the project object @param parent parent widget of this browser (QWidget) """ self.omniidl = Preferences.getCorba("omniidl") if self.omniidl == "": self.omniidl = Utilities.isWindowsPlatform() and \ "omniidl.exe" or "omniidl" if not Utilities.isinpath(self.omniidl): self.omniidl = None ProjectBaseBrowser.__init__(self, project, ProjectBrowserInterfaceType, parent) self.selectedItemsFilter = \ [ProjectBrowserFileItem, ProjectBrowserSimpleDirectoryItem] self.setWindowTitle(self.tr('Interfaces (IDL)')) self.setWhatsThis(self.tr( """<b>Project Interfaces Browser</b>""" """<p>This allows to easily see all interfaces (CORBA IDL files)""" """ contained in the current project. Several actions can be""" """ executed via the context menu.</p>""" )) project.prepareRepopulateItem.connect(self._prepareRepopulateItem) project.completeRepopulateItem.connect(self._completeRepopulateItem)
def on_addButton_clicked(self): """ Private slot to add a new idntity. """ name, ok = QInputDialog.getText( self, self.tr("Add Identity"), self.tr("Identity Name:"), QLineEdit.Normal) if ok: if name: if name in self.__identities: E5MessageBox.critical( self, self.tr("Add Identity"), self.tr( """An identity named <b>{0}</b> already exists.""" """ You must provide a different name.""").format( name)) self.on_addButton_clicked() else: identity = IrcIdentity(name) identity.setIdent(Utilities.getUserName()) identity.setRealName(Utilities.getRealName()) self.__identities[name] = identity self.identitiesCombo.addItem(name) self.identitiesCombo.setCurrentIndex( self.identitiesCombo.count() - 1) else: E5MessageBox.critical( self, self.tr("Add Identity"), self.tr("""The identity has to have a name.""")) self.on_addButton_clicked()
def __advance(self): """ Private method to advance to the next error. """ try: next(self.__spell) except StopIteration: self.__enableButtons(False) self.contextLabel.setText("") self.changeEdit.setText("") self.suggestionsList.clear() return self.__enableButtons(True) self.word, self.wordStart, self.wordEnd = self.__spell.getError() lcontext, rcontext = self.__spell.getContext( self.wordStart, self.wordEnd) self.changeEdit.setText(self.word) self.contextLabel.setText( '{0}<font color="#FF0000">{1}</font>{2}'.format( Utilities.html_encode(lcontext), self.word, Utilities.html_encode(rcontext))) suggestions = self.__spell.getSuggestions(self.word) self.suggestionsList.clear() self.suggestionsList.addItems(suggestions)
def stressTest(): """ Stress Test function """ responseDict = { } # Do stuff os.chdir('./stress') p = subprocess.Popen(['fl-run-test','test_Stress.py'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) err, out = p.communicate() length = len(out.split('\n')) #print out.split('\n')[length-2] if "FAILED" in out.split('\n')[length-2] : u.printError('URL not reachable...') u.printError('Stress testing aborted...') responseDict['status'] = False else : u.printSuccess('URL reachable...') u.printInfo('Stress testing initiated...') p = subprocess.Popen(['fl-run-bench','test_Stress.py','Stress.test_stress'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) p.communicate() #out,err = p.communicate() #print out.split('\n') p = subprocess.Popen(['fl-build-report','--html','stress-bench.xml'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, error = p.communicate() #returnVal = p.communicate() # get the path of the generated report... path = out.split('\n')[1] u.printInfo('Report stored at '+path) responseDict['status'] = True responseDict['path'] = path os.chdir('..') return responseDict
def __buildModulesDict(self): """ Private method to build a dictionary of modules contained in the package. @return dictionary of modules contained in the package. """ moduleDict = {} modules = glob.glob(Utilities.normjoinpath(self.packagePath,'*.py')) + \ glob.glob(Utilities.normjoinpath(self.packagePath,'*.pyw')) + \ glob.glob(Utilities.normjoinpath(self.packagePath,'*.ptl')) tot = len(modules) progress = KQProgressDialog(self.trUtf8("Parsing modules..."), QString(), 0, tot, self) try: prog = 0 progress.show() QApplication.processEvents() for module in modules: progress.setValue(prog) QApplication.processEvents() prog = prog + 1 try: mod = Utilities.ModuleParser.readModule(module, caching = False) except ImportError: continue else: name = mod.name if name.startswith(self.package): name = name[len(self.package) + 1:] moduleDict[name] = mod finally: progress.setValue(tot) return moduleDict
def drawSkewedCoordinateSystem(context): # alpha is 22.5 degrees and beta is 15 degrees. alpha = math.pi/8 beta = math.pi/12 # Create a rectangle that is 72 units on a side # with its origin at (0,0). r = Quartz.CGRectMake(0, 0, 72, 72) Quartz.CGContextTranslateCTM(context, 144, 144) # Draw the coordinate axes untransformed. Utilities.drawCoordinateAxes(context) # Fill the rectangle. Quartz.CGContextFillRect(context, r) # Create an affine transform that skews the coordinate system, # skewing the x-axis by alpha radians and the y-axis by beta radians. skew = Quartz.CGAffineTransformMake(1, math.tan(alpha), math.tan(beta), 1, 0, 0) # Apply that transform to the context coordinate system. Quartz.CGContextConcatCTM(context, skew) # Set the fill and stroke color to a dark blue. Quartz.CGContextSetRGBStrokeColor(context, 0.11, 0.208, 0.451, 1) Quartz.CGContextSetRGBFillColor(context, 0.11, 0.208, 0.451, 1) # Draw the coordinate axes again, now transformed. Utilities.drawCoordinateAxes(context) # Set the fill color again but with a partially transparent alpha. Quartz.CGContextSetRGBFillColor(context, 0.11, 0.208, 0.451, 0.7) # Fill the rectangle in the transformed coordinate system. Quartz.CGContextFillRect(context, r)
def __createServerCertificateEntry(self, server, cert): """ Private method to create a server certificate entry. @param server server name of the certificate (string) @param cert certificate to insert (QSslCertificate) """ # step 1: extract the info to be shown if qVersion() >= "5.0.0": organisation = Utilities.decodeString(", ".join(cert.subjectInfo(QSslCertificate.Organization))) commonName = Utilities.decodeString(", ".join(cert.subjectInfo(QSslCertificate.CommonName))) else: organisation = Utilities.decodeString(cert.subjectInfo(QSslCertificate.Organization)) commonName = Utilities.decodeString(cert.subjectInfo(QSslCertificate.CommonName)) if organisation is None or organisation == "": organisation = self.tr("(Unknown)") if commonName is None or commonName == "": commonName = self.tr("(Unknown common name)") expiryDate = cert.expiryDate().toString("yyyy-MM-dd") # step 2: create the entry items = self.serversCertificatesTree.findItems(organisation, Qt.MatchFixedString | Qt.MatchCaseSensitive) if len(items) == 0: parent = QTreeWidgetItem(self.serversCertificatesTree, [organisation]) else: parent = items[0] itm = QTreeWidgetItem(parent, [commonName, server, expiryDate]) itm.setData(0, self.CertRole, cert.toPem())
def run_server(port): """ Main function to run the server. :param port: the port number which the server should run on :return: """ # load config file for DH and username/password global crypto_service, password_hash_dict, chatting_service, user_addr_dict, auth_dict g = c.DH_GENERATOR p = util.load_df_param_from_file(c.DH_CONFIG_PATH) crypto_service = CryptoService(rsa_pri_path=c.PRI_KEY_PATH, p=p, g=g) chatting_service = ChattingService(user_addr_dict, auth_dict, crypto_service) password_hash_dict = util.load_pickle_file(c.PW_HASH_PATH) try: local_ip = socket.gethostbyname(socket.gethostname()) print('Binding to ip: {}'.format(local_ip)) serv_addr = (local_ip, port) serv = SocketServer.UDPServer(serv_addr, ClientRequestHandler) # dump the server address to config file for client using util.save(c.SERVER_CONFIG_PATH, serv_addr, True) except socket.error: print c.FAIL_SRV_INIT return print c.SRV_START try: serv.serve_forever() except KeyboardInterrupt: # Stop server when seeing ctrl-c pass except: print c.FAIL_SRV_START finally: serv.server_close()
def visit(self, line): resultLine = '' lastIndex = 0 template = re.compile("{|}") rezults = template.finditer(line) indentSize = Utilities.getIndentSize(line) firstChar = Utilities.findNextNonWhiteSpaceCharIndex(line, 0) for rezult in rezults: index = rezult.start() if not(Utilities.isInsideTextLiteral(line, index) or index == firstChar): resultLine += line[lastIndex:index] resultLine += '\n' if(line[index] == '{'): resultLine += ' ' * indentSize indentSize += 4 else: indentSize -= 4 resultLine += ' ' * indentSize resultLine += line[index] resultLine += '\n' lastIndex = Utilities.findNextNonWhiteSpaceCharIndex(line, index + 1 ) if(lastIndex != -1): resultLine += ' ' * indentSize if(lastIndex != -1): resultLine += line[lastIndex:] return resultLine
def main(): robot = (1,1) dest = (-1,-1) opp = (0,.2) perp_point = get_perpendicular_point(robot, dest, opp) if perp_point is None: print "notevenclose" if do_i_need_to_avoid(opp, perp_point): # not sure what to do right now go_above = above_or_below(robot, dest, opp) # eventually do this crit_point = point_to_go_through(opp, perp_point) angle = Utilities.get_angle_between_points(robot[0], robot[1], crit_point[0], crit_point[1]) c = Utilities.get_distance_between_points(robot[0], robot[1], dest[0], dest[1]) a = c*np.cos(angle) b = c*np.sin(angle) x = robot[0] + a y = robot[1] + b print (x,y) else: print "do not need to avoid"
def drawColoredLogo(context): r = CGRectMake(0, 0, 100, 100) CGContextSaveGState(context) if 1: # Position the center of the rectangle on the left. CGContextTranslateCTM(context, 140, 140) # Rotate so that the rectangles are rotated 45 degrees # about the current coordinate origin. CGContextRotateCTM(context, Utilities.DEGREES_TO_RADIANS(45)) # Translate so that the center of the rect is at the previous origin. CGContextTranslateCTM(context, -r.size.width/2, -r.size.height/2) # Set the fill color to a purple color. CGContextSetFillColorWithColor(context, Utilities.getRGBOpaquePurpleColor()) # Fill the first rectangle. CGContextFillRect(context, r) # Position to draw the right-most rectangle. CGContextTranslateCTM(context, 60, -60) # Set the fill color to a yellow color. CGContextSetFillColorWithColor(context, Utilities.getRGBOpaqueYellowColor()) CGContextFillRect(context, r) # Position for the center rectangle. CGContextTranslateCTM(context, -30, +30) # Set the stroke color to an orange color. CGContextSetStrokeColorWithColor(context, Utilities.getRGBOpaqueOrangeColor()) # Stroke the rectangle with a linewidth of 12. CGContextStrokeRectWithWidth(context, r, 12) CGContextRestoreGState(context)
def calc_arvo_geo_measures(self, radius): """ Calculates geometrical measures: volume, area """ _temp_file = Utilities.get_random_name() _temp_file = "temp" # TODO: Need a way set the surface radius for a each atom. Probably add it as # as a new column in the atoms.in file. # prepare a temporary file self._writeATS(_temp_file, radius) command = "%s protein=%s" % ("/Users/Tomas/git/Analysis-Toolkit/thirdparty/arvo_c/arvo_c", _temp_file) output, stderr, status = Utilities.run_sub_process(command) # if the execution of the was successful: if not status: output_array = output.split() self.arvo_calc = True self.arvo_volume = np.float64(output_array[1]) self.arvo_area = np.float64(output_array[3]) self.arvo_spheres = np.int16(output_array[6]) os.unlink(_temp_file)
def __init__(self): screen = pygame.display.get_surface() self.area = screen.get_rect() self.fitness = random.randint(5,15) self.awareness = 70 + self.fitness self.speed = 50 + self.fitness self.radius = self.awareness # self.theta = 0 # self.thetaVelocity = 1 self.active = True self.target = None self.med = False self.high = False self.medFitness = Utilities.load_image('hunter2.png') self.highFitness = Utilities.load_image('hunter1.png') if(self.fitness >= 30): self.image, self.rect = self.highFitness self.high = True elif(self.fitness >= 5): self.image, self.rect = self.medFitness self.med = True self.tempTarget = (self.area.centerx, self.area.centery) self.rect.move(50,50) pygame.sprite.Sprite.__init__(self) print "CREATED HUNTER WITH FITNESS " + str(self.fitness) + " and speed " + str(self.speed) + " and awareness " + str(self.awareness)
def peakDetectionTest(path=None, noise=0.08): """Use pre-processing funcs to detect peaks""" if path == None: path = "testfiles" names = Utilities.createRandomStrings(8, 6) fname = os.path.join(path, "spectraldata.txt") peaks = Utilities.createSimulatedSpectralData(fname, names, noise=noise) conf = { "format": "databycolumn", "saveplots": 1, "marker": "-", "markers": "-,x", "alpha": 0.7, "normalise": 1, "function1": "smooth", "function2": "baselinecorrection", "function3": "detectpeaks", } p = Pipeline() p.createConfig("temp.conf", **conf) p.openRaw(fname) results = p.run() # compare predicted peaks successrates = [] res = results[results.keys()[0]] for name in peaks: # print name, sorted(peaks[name]), results[name][0] orig = set(peaks[name]) pred = set(res[name][0]) s = float(len(orig.intersection(pred))) / (len(orig)) successrates.append(s) return np.mean(successrates)
def _getDict(cls, lang, pwl = "", pel = ""): """ Protected classmethod to get a new dictionary. @param lang the language to be used as the default (string). The string should be in language locale format (e.g. en_US, de). @keyparam pwl name of the personal/project word list (string) @keyparam pel name of the personal/project exclude list (string) @return reference to the dictionary (enchant.Dict) """ if not pwl: pwl = unicode(Preferences.getEditor("SpellCheckingPersonalWordList")) if not pwl: pwl = os.path.join(Utilities.getConfigDir(), "spelling", "pwl.dic") d = os.path.dirname(pwl) if not os.path.exists(d): os.makedirs(d) if not pel: pel = unicode(Preferences.getEditor("SpellCheckingPersonalExcludeList")) if not pel: pel = os.path.join(Utilities.getConfigDir(), "spelling", "pel.dic") d = os.path.dirname(pel) if not os.path.exists(d): os.makedirs(d) try: d = enchant.DictWithPWL(lang, pwl, pel) except: # Catch all exceptions, because if pyenchant isn't available, you # can't catch the enchant.DictNotFound error. d = None return d
def on_diffButton_clicked(self): """ Private slot to handle the Compare button press. """ filename1 = Utilities.toNativeSeparators(self.file1Edit.text()) try: f1 = open(filename1, "r", encoding="utf-8") lines1 = f1.readlines() f1.close() except IOError: E5MessageBox.critical( self, self.tr("Compare Files"), self.tr( """<p>The file <b>{0}</b> could not be read.</p>""") .format(filename1)) return filename2 = Utilities.toNativeSeparators(self.file2Edit.text()) try: f2 = open(filename2, "r", encoding="utf-8") lines2 = f2.readlines() f2.close() except IOError: E5MessageBox.critical( self, self.tr("Compare Files"), self.tr( """<p>The file <b>{0}</b> could not be read.</p>""") .format(filename2)) return self.__compare(lines1, lines2)
def gather_pb8k(file_path, variances, N): class ParseCallback(object): def __init__(self, variance, N): self.variance = variance self.N = N self.index = {} self.results = {} def __call__(self, line_number, comps): if line_number==0: return gene = comps[PB8KFileInfo.TF1.HUGO] if "," in gene: multiple_genes = gene.split(",") for g in multiple_genes: row = row_from_comps(comps, PB8KFileInfo.TF1, self.variance, self.N) self.process_row(g, row) else: row = row_from_comps(comps, PB8KFileInfo.TF1, self.variance, self.N) self.process_row(gene, row) def process_row(self, gene, row): if row is None: return snp = row[1] self.index[snp] = gene if not gene in self.results: self.results[gene] = [] self.results[gene].append(row) callback = ParseCallback(variances, N) Utilities.parse_file(file_path, callback, expect_throw=True) return callback.index, callback.results
def myDrawRedBlackCheckerBoardPattern(info, patternCellContext): # This pattern proc draws a red and a black rectangle # patch representing the minimum cell needed to paint a # checkerboard with that pattern. # # Each 'cell' of the checkerboard is 2 units on a side. # # This code uses CGColorRefs which are available in Panther # and later only. Patterns are available in all versions of # Mac OS X but this code uses CGColorRefs for convenience # and efficiency. # Paint a black checkerboard box. CGContextSetFillColorWithColor(patternCellContext, Utilities.getRGBOpaqueBlackColor()) # This is a 1x1 unit rect whose origin is at 0,0 in pattern space. CGContextFillRect(patternCellContext, CGRectMake(0.0, 0.0, 1.0, 1.0)) # This is a 1x1 unit rect whose origin is at 1,1 in pattern space. CGContextFillRect(patternCellContext, CGRectMake(1.0, 1.0, 1.0, 1.0)) # Paint a red checkerboard box. CGContextSetFillColorWithColor(patternCellContext, Utilities.getRGBOpaqueRedColor()) # This is a 1x1 unit rect whose origin is at 1,0 in pattern space, # that is, immediately to the right of first black checkerboard box. CGContextFillRect(patternCellContext, CGRectMake(1.0, 0.0, 1.0, 1.0)) # This is a 1x1 unit rect whose origin is at 0,1 in pattern space, # that is, immediately above the first black checkerboard box. CGContextFillRect(patternCellContext, CGRectMake(0.0, 1.0, 1.0, 1.0))
def detectMissingModules(folder, outputFile): #print("%s" % (inspect.stack()[0][3])) classNamesSet = set() includeClassSet = set() for abspath, dirs, files in os.walk(folder): for file in files: #print(file) if file.endswith(".pp") and not os.path.islink(os.path.join(abspath, file)): #print(file) fileObj = SourceModel.SM_File.SM_File(os.path.join(abspath, file)) classNames, fileIncludes = detectMissingClassesByInclude(fileObj, outputFile) #print("Classes: %s" % ','.join(n for n in classNames)) classNamesSet = classNamesSet.union(classNames) #print("Union with %s: %s" % (','.join(n for n in classNames), ','.join(n for n in classNamesSet))) includeClassSet = includeClassSet.union(fileIncludes) #print("%s: Classes: %s" % (inspect.stack()[0][3], ','.join(n for n in classNamesSet))) #print("%s: Class includes: %s" % (inspect.stack()[0][3], ','.join(i for i in includeClassSet))) missingDependencySet = includeClassSet.difference(classNamesSet) #print(includeClassSet) #print(classNamesSet) #print(missingDependencySet) Utilities.myPrint("Missing dependency set: %s" % ','.join(c for c in missingDependencySet)) #for md in missingDependencySet: #Utilities.reportSmell(outputFile, folder, CONSTS.SMELL_MIS_DEP, CONSTS.FILE_RES) with open('missDependencies.puppeteer.txt', 'a+') as f: for md in missingDependencySet: f.write("%s\n" % md) Utilities.reportSmell(outputFile, folder, CONSTS.SMELL_MIS_DEP, CONSTS.FILE_RES)
def build(self, features, labels, lamda, term_fun, thresh, is_batch=True): # construct x with the bias column x = [[1] + f.tolist() for f in features] x = np.array(x) y = np.array([[l] for l in labels]) # initialize the theta and iteration counter # theta = np.zeros((len(x[0]), 1)) theta = np.array([[random.random()] for i in range(len(x[0]))]) self.iter_count = 0 acc_count = [0, 0] # recursively update theta while not term_fun(theta, features, y, thresh, acc_count): # while not term_fun(theta, features, y, self.accs, thresh): if is_batch: hx = np.array(util.logistic_fun_batch(theta, features)) diffs = y - hx for j in range(len(theta)): sum = 0 for i in range(len(diffs)): sum += diffs[i][0] * x[i][j] theta[j][0] = theta[j][0] + lamda * sum else: for i in range(len(x)): hx = util.logistic_fun(theta, x[i]) diff = y[i][0] - hx for j in range(len(theta)): theta[j][0] = theta[j][0] + lamda * diff * x[i][j] self.iter_count += 1 self.theta = theta
def drawWithClippingMask(context, theMaskingImageURL, maskwidth, maskheight): # An array of CGColor objects. colors = ( Utilities.getRGBOpaqueDarkGreenColor(), Utilities.getRGBOpaqueDarkBlueColor(), Utilities.getRGBOpaqueBlueColor(), Utilities.getRGBOpaqueRedColor()) maskBitsPerComponent = 8 bytesPerRow = ( (maskwidth * maskBitsPerComponent) + 7)/8 shouldInterpolate = True maskDataProvider = CGDataProviderCreateWithURL(theMaskingImageURL) if maskDataProvider is None: print >>sys.stderr, "Couldn't create Image Mask provider!" return mask = CGImageMaskCreate(maskwidth, maskheight, maskBitsPerComponent, maskBitsPerComponent, maskwidth, maskDataProvider, None, shouldInterpolate) del maskDataProvider if mask is None: print >>sys.stderr, "Couldn't create Image Mask!" return maskRect = CGRectMake(0, 0, maskwidth/3, maskheight/3) # Position for drawing the mask at the left side of the figure. CGContextTranslateCTM(context, 50, 50 ) # Set the context fill color to a CGColor object that is black. CGContextSetFillColorWithColor(context, getRGBOpaqueBlackColor()) # Draw the mask. It is painted with with the black fill color. CGContextDrawImage(context, maskRect, mask) # Position to the right of the mask just painted. CGContextTranslateCTM(context, CGRectGetWidth(maskRect) + 25, 0) # Clip to the mask. CGContextClipToMask(context, maskRect, mask) # Release the mask since this code no longer needs it. del mask # Make a rect that has a width and height 1/3 that of the image mask. rect = CGRectMake(0, 0, CGRectGetWidth(maskRect)/3, CGRectGetHeight(maskRect)/3) CGContextTranslateCTM(context, 0, 2*CGRectGetHeight(rect)) # Draw a 3 x 3 grid of rectangles, setting the color for each rectangle # by cycling through the array of CGColor objects in the 'colors' array. for j in range(3): CGContextSaveGState(context) for i in range(3): # Draw a row of rectangles. # Set the fill color using one of the CGColor objects in the # colors array. CGContextSetFillColorWithColor(context, colors[(i+j) % 4]) CGContextFillRect(context, rect) CGContextTranslateCTM(context, CGRectGetWidth(rect), 0) CGContextRestoreGState(context) # Position to draw the next row. CGContextTranslateCTM(context, 0, -CGRectGetHeight(rect))
def printAccessionNumbersFromName(filePath,dbFolderPath,outFolder): file = open(filePath,"r") o = open(outFolder+"/filter.txt","w") org = [] accessions = [] for line in file: temp = '_'.join(line.split("_")[:2]) org.append(temp.strip()) for dir in util.return_recursive_dir_files(dbFolderPath): dirSplit = dir.split("/") organism = "" accession = "" for f in util.return_recursive_files(dir): #print f seq_record = SeqIO.parse(open(f), "genbank").next() accession = seq_record.annotations['accessions'][0] organism_tmp = seq_record.annotations['organism'].replace(' ', '_') organism_tmp_1 = re.sub('[\[\]]', "", organism_tmp) organism = '_'.join(organism_tmp_1.split('_')[:2])#+"_"+accession try: if(org.index(organism) >= 0): accessions.append(accession) except: #print "none" pass for accesion in accessions: o.write(accesion+"\n") o.close() file.close()
def __findDuplicates(self, cond, special, showMessage=False, index=QModelIndex()): """ Private method to check, if an entry already exists. @param cond condition to check (string) @param special special condition to check (string) @param showMessage flag indicating a message should be shown, if a duplicate entry is found (boolean) @param index index that should not be considered duplicate (QModelIndex) @return flag indicating a duplicate entry (boolean) """ idx = self.__model.getWatchPointIndex(cond, special) duplicate = idx.isValid() and \ idx.internalPointer() != index.internalPointer() if showMessage and duplicate: if not special: msg = self.tr("""<p>A watch expression '<b>{0}</b>'""" """ already exists.</p>""")\ .format(Utilities.html_encode(cond)) else: msg = self.tr( """<p>A watch expression '<b>{0}</b>'""" """ for the variable <b>{1}</b> already exists.</p>""")\ .format(special, Utilities.html_encode(cond)) E5MessageBox.warning( self, self.tr("Watch expression already exists"), msg) return duplicate
def replicatesTest(): """Tests handling of replicates""" p = Pipeline() conf = { "format": "databycolumn", "groupbyname": 1, "parsenamesindex": 1, "parsemethod": "numeric", "replicates": 1, "model1": "linear", "variable1": "a", "model2": "sigmoid", "variable2": "tm", } p.createConfig("temp.conf", **conf) reps = ["rep1", "rep2", "rep3"] path = "testfiles/replicates" Utilities.createDirectory(path) names = Utilities.createRandomStrings(3, 6) for r in reps: rpath = os.path.join(path, r) Utilities.createGroupedData(rpath, names=names) p.addFolder(path) p.run() return
def prepare_image(self, n): self.second_image = [] #read in image n path = self.directory + Constants.working_directory + Constants.image_directory + Constants.reduced_prefix temp_image = Utilities.get_image_data(path, n) set, i = Utilities.n_to_set_and_n(n) file = path + Constants.file_name + "_" + str( set) + "_" + Utilities.format_index(i) + Constants.fits_extension head = getheader(file, ignore_missing_end=True) head = getheader(path + Constants.file_name + "_" + str(set) + "_" + Utilities.format_index(i) + Constants.fits_extension, ignore_missing_end=True) #read in the image n-1 first_image = Utilities.get_image_data(path, n - 1) #calculate the shift between the these two images x_shift = self.x_shifts[n - 1] - self.x_shifts[n - 2] y_shift = self.y_shifts[n - 1] - self.y_shifts[n - 2] #calculate the mean of the preceding image mean = np.mean(first_image) for i in range(len(temp_image)): for j in range(len(temp_image[0])): #find the expected position of pixel (j, i) from image n in #image n-1. This is done to align the images y = int(i - round(y_shift)) x = int(j - round(x_shift)) #if the expected position is within the bounds of the image if x > 0 and y > 0 and x < len( temp_image[0]) and y < len(temp_image): #divide the count stored in pixel (j, i) by the count stored #in the equivalent pixel in the preceding image divided by the #mean of the preceding image. temp_image[i][j] = temp_image[i][j] / (first_image[y][x] / mean) #calculate median of image n median = np.median(temp_image) #loop through each pixel in image n for i in range(len(temp_image)): #prepare array to store the final version of image n for scanning self.second_image.append([]) for j in range(len(temp_image[0])): #if the count in pixel (j, i) is greater than the median if temp_image[i][j] > median: #stores the number of pixels surrounding this pixel #that have a count higher than the median count = 0 #find number of surrounding pixels which have a count higher #than the median for k in range(-1, 2): for l in range(-1, 2): if not (k == 0 and l == 0): ik = i + k jl = j + l if ik >= 0 and jl >= 0 and ik < len( temp_image) and jl < len( temp_image[0]): if temp_image[ik][jl] > median: count += 1 #if the number of surrounding pixels with a count higher #than the median is less that 3, append the median rather #than the actual pixel value to the final version of the image if count < 3: self.second_image[i].append(median) else: self.second_image[i].append(temp_image[i][j]) #pixels with a count lower than the median are automatically #inserted into the final version else: self.second_image[i].append(temp_image[i][j])
def slow_print_self(self): Util.slow_print(str(self))
def readmodule_ex(module, path=[]): """ Read a CORBA IDL file and return a dictionary of classes, functions and modules. @param module name of the CORBA IDL file (string) @param path path the file should be searched in (list of strings) @return the resulting dictionary """ global _modules dict = {} dict_counts = {} if module in _modules: # we've seen this file before... return _modules[module] # search the path for the file f = None fullpath = list(path) f, file, (suff, mode, type) = ClassBrowsers.find_module(module, fullpath) if f: f.close() if type not in SUPPORTED_TYPES: # not CORBA IDL source, can't do anything with this module _modules[module] = dict return dict _modules[module] = dict classstack = [] # stack of (class, indent) pairs indent = 0 try: src = Utilities.readEncodedFile(file)[0] except (UnicodeError, IOError): # can't do anything with this module _modules[module] = dict return dict lineno, last_lineno_pos = 1, 0 lastGlobalEntry = None cur_obj = None i = 0 while True: m = _getnext(src, i) if not m: break start, i = m.span() if m.start("Method") >= 0: # found a method definition or function thisindent = indent meth_name = m.group("MethodName") meth_sig = m.group("MethodSignature") meth_sig = meth_sig and meth_sig.replace('\\\n', '') or '' meth_sig = _commentsub('', meth_sig) meth_sig = _normalize(' ', meth_sig) lineno = lineno + src.count('\n', last_lineno_pos, start) last_lineno_pos = start # close all interfaces/modules indented at least as much while classstack and \ classstack[-1][1] >= thisindent: if classstack[-1][0] is not None: # record the end line classstack[-1][0].setEndLine(lineno - 1) del classstack[-1] if classstack: # it's an interface/module method cur_class = classstack[-1][0] if isinstance(cur_class, Interface) or \ isinstance(cur_class, Module): # it's a method f = Function(None, meth_name, file, lineno, meth_sig) cur_class._addmethod(meth_name, f) # else it's a nested def else: f = None else: # it's a function f = Function(module, meth_name, file, lineno, meth_sig) if meth_name in dict_counts: dict_counts[meth_name] += 1 meth_name = "{0}_{1:d}".format(meth_name, dict_counts[meth_name]) else: dict_counts[meth_name] = 0 dict[meth_name] = f if not classstack: if lastGlobalEntry: lastGlobalEntry.setEndLine(lineno - 1) lastGlobalEntry = f if cur_obj and isinstance(cur_obj, Function): cur_obj.setEndLine(lineno - 1) cur_obj = f classstack.append((f, thisindent)) # Marker for nested fns elif m.start("String") >= 0: pass elif m.start("Comment") >= 0: pass elif m.start("Interface") >= 0: # we found an interface definition thisindent = indent indent += 1 # close all interfaces/modules indented at least as much while classstack and \ classstack[-1][1] >= thisindent: if classstack[-1][0] is not None: # record the end line classstack[-1][0].setEndLine(lineno - 1) del classstack[-1] lineno = lineno + src.count('\n', last_lineno_pos, start) last_lineno_pos = start class_name = m.group("InterfaceName") inherit = m.group("InterfaceSupers") if inherit: # the interface inherits from other interfaces inherit = inherit[1:].strip() inherit = [_commentsub('', inherit)] # remember this interface cur_class = Interface(module, class_name, inherit, file, lineno) if not classstack: dict[class_name] = cur_class else: cls = classstack[-1][0] cls._addclass(class_name, cur_class) if not classstack: if lastGlobalEntry: lastGlobalEntry.setEndLine(lineno - 1) lastGlobalEntry = cur_class if cur_obj and isinstance(cur_obj, Function): cur_obj.setEndLine(lineno - 1) cur_obj = cur_class classstack.append((cur_class, thisindent)) elif m.start("Module") >= 0: # we found a module definition thisindent = indent indent += 1 # close all interfaces/modules indented at least as much while classstack and \ classstack[-1][1] >= thisindent: if classstack[-1][0] is not None: # record the end line classstack[-1][0].setEndLine(lineno - 1) del classstack[-1] lineno = lineno + src.count('\n', last_lineno_pos, start) last_lineno_pos = start module_name = m.group("ModuleName") # remember this module cur_class = Module(module, module_name, file, lineno) if not classstack: dict[module_name] = cur_class if lastGlobalEntry: lastGlobalEntry.setEndLine(lineno - 1) lastGlobalEntry = cur_class if cur_obj and isinstance(cur_obj, Function): cur_obj.setEndLine(lineno - 1) cur_obj = cur_class classstack.append((cur_class, thisindent)) elif m.start("Attribute") >= 0: lineno = lineno + src.count('\n', last_lineno_pos, start) last_lineno_pos = start index = -1 while index >= -len(classstack): if classstack[index][0] is not None and \ not isinstance(classstack[index][0], Function) and \ not classstack[index][1] >= indent: attributes = m.group("AttributeNames").split(',') ro = m.group("AttributeReadonly") for attribute in attributes: attr = Attribute(module, attribute, file, lineno) if ro: attr.setPrivate() classstack[index][0]._addattribute(attr) break else: index -= 1 if lastGlobalEntry: lastGlobalEntry.setEndLine(lineno - 1) lastGlobalEntry = None elif m.start("Begin") >= 0: # a begin of a block we are not interested in indent += 1 elif m.start("End") >= 0: # an end of a block indent -= 1 else: assert 0, "regexp _getnext found something unexpected" return dict
def screen_height(self, height): return Utilities.checkDimensions(height)
def screen_width(self, width): return Utilities.checkDimensions(width)
def output_results(self): #get time of observation for each image times_file = self.directory + Constants.working_directory + Constants.time_file times = [line.rstrip('\n') for line in open(times_file)] #make streak folder in results folder output_dir = self.directory + Constants.working_directory + Constants.output_directory + Constants.streak_folder if not os.path.exists(output_dir): os.mkdir(output_dir) #path to which the table fo streaks is written streak_file = output_dir + Constants.streak_file f = open(streak_file, "w") f.write( "id time xcentroid ycentroid ra dec RA/s Dec/s " ) #stores the number of consecutive images with streaks count = 0 #identifier for streak objid = 1 #ra and dec for an individual streak ras = [] decs = [] #time each image was taken for a consecutive set of images which contains #a streak streak_times = [] #loop through array of streaks found. This array contains an x and y position #for the centre of each streak in each image (one element for each image). #If no streaks were found in that image, the element is empty. #Here we are checking for a consecutive set of images which contain a streak. for i in range(len(self.streaks)): #if image i contains a streak if len(self.streaks[i]) > 0: count += 1 #if the streak is at least 150 pixels away from the edge of the image, #record its x and y position and the time at which it was at #that position. For use in calculating object velocity if Utilities.is_within_boundaries(self.streaks[i][0][0], self.streaks[i][0][1], Constants.image_width, Constants.image_height, 150): time_string = times[i] time_units = time_string.split("T") time_units = time_units[0].split( "-") + time_units[1].split(":") time = datetime.datetime(int(time_units[0]), int(time_units[1]), int(time_units[2]), int(time_units[3]), int(time_units[4]), int(time_units[5])) #add half the exposure time to the time of observation. #This is done because the positions stored are at the centre #of the streak. The object is at this position halfway #through the exposure . time = time + datetime.timedelta(0, 17.5) streak_times.append(time) #use world coordinate system from Cataloguer to find #the RA and Dec of the centre of the streak ra, dec = self.cataloguer.wcs.all_pix2world( self.streaks[i][0][0], self.streaks[i][0][1], 0) ras.append(ra) decs.append(dec) #if image i does not contain a streak, or the end of the array has been reached if len(self.streaks[i]) == 0 or i == len(self.streaks) - 1: #if the minimum number of consecutive images with a streak has #been reached if count > 3: #find a time associated with one of the images and the #ra and dec of the centre of the streak in that image time = streak_times[len(streak_times) - 2] ra, dec = self.cataloguer.wcs.all_pix2world( self.streaks[i - 2][0][0], self.streaks[i - 2][0][1], 0) #prepare string for output output_string = "\r\n" + Utilities.format_index( objid ) + " " + time.strftime("%m/%d/%Y, %H:%M:%S") + " " + str( self.streaks[i - 2][0][0]) + " " + str(self.streaks[ i - 2][0][1]) + " " + str(ra) + " " + str(dec) avg_ra_velocity = 0 avg_dec_velocity = 0 #calculate the angular velocity in terms of RA/s and Dec/s #in arcseconds/s by finding the mean change in RA and Dec of #the centre of the streak in consecutive images and dividing #this by the exposure time for j in range(len(ras) - 1): time1 = streak_times[j] time2 = streak_times[j + 1] diff = (time2 - time1).total_seconds() avg_ra_velocity += (ras[j + 1] - ras[j]) / diff avg_dec_velocity += (decs[j + 1] - decs[j]) / diff avg_ra_velocity = avg_ra_velocity / (len(ras) - 1) avg_dec_velocity = avg_dec_velocity / (len(decs) - 1) #add angular velocities to output string (*3600 to convert to arcseconds/s) output_string += " " + str( avg_ra_velocity * 3600) + " " + str( avg_dec_velocity * 3600) #write output string to file f.write(output_string) #create and save thumnbnail of the streak in the results/streaks folder im = self.ff.get_thumbnail(i - 2, self.streaks[i - 3][0][0], self.streaks[i - 3][0][1], 100, False) plt.axis('off') plt.imshow(im, origin='lower', cmap=plt.cm.inferno) plt.savefig(output_dir + "streak_" + Utilities.format_index(objid) + ".jpg") #increment object id (so that each streak has a unique ID) objid += 1 #since the current image has no streak, or the end of the streak #array has been reached, the number of consecutive images is #reset to zero count = 0 x_pos = [] y_pos = [] streak_times = []
def find_streaks(self, n): #stores x and y positions of the centres of any streaks found in the #image streaks = [] #stores the pixels that make up each streak streak_pixels = [] #refer to image n as 'data' (for simplicity) data = self.second_image median = np.median(data) #stores all pixels with counts greater than the median which have been scanned all_pixels = set() #stores all pixels which form streaks in the image all_streaks = set() #loop through each pixel in the image for i in range(len(data)): for j in range(len(data[0])): #if the count in the pixel is greater than 1.04 times the median #and it has not been scanned already if data[i][j] > median * 1.04 and not string in all_pixels: #form string storing x and y position of pixel, to be stored #in the various sets of pixels. Cannot simply use an array #as these cannot be inserted into a set. string = str(i) + " " + str(j) completed = False to_scan = [] pixels = [] pixels.append(string) to_scan.append(string) all_pixels.add(string) while not completed: arr = to_scan[0].split(" ") y = int(arr[0]) x = int(arr[1]) found_pixels = self.find_pixels( x, y, data, median, all_pixels) if len(pixels) > 2000: completed = True pixels = pixels + found_pixels to_scan = to_scan + found_pixels to_scan.pop(0) if len(to_scan) == 0: completed = True if len(pixels) > 60: x_centre = 0 y_centre = 0 max_dist = 0 for i in range(len(pixels)): arr = pixels[i].split(" ") y = int(arr[0]) x = int(arr[1]) x_centre += x y_centre += y for k in range(i, len(pixels)): if not k == i: arr = pixels[k].split(" ") y2 = int(arr[0]) x2 = int(arr[1]) dist = ((x2 - x)**2 + (y2 - y)**2)**0.5 if dist > max_dist: max_dist = dist x_centre = int(round(x_centre / len(pixels))) y_centre = int(round(y_centre / len(pixels))) if not Utilities.is_within_boundaries( x_centre, y_centre, len(data[0]), len(data), 30): continue vector = [1, 0] occupancies = [] max_occupancy = 0 max_vector = 0 for angle in range(0, 180): count = 0 line_pixels = set() for a in range(-30, 31): for b in range(-30, 31): y = y_centre + a x = x_centre + b if self.distance_to_line( [x_centre, y_centre], vector, [x, y]) < 3: line_pixels.add(str(y) + " " + str(x)) for pix in line_pixels: if pix in pixels: count = count + 1 occupancy = count / len(line_pixels) occupancies.append(occupancy) # if occupancy > max_occupancy: # max_occupancy = occupancy # max_vector = vector vector = self.rotate(vector, 1) standard_deviation = Utilities.standard_deviation( occupancies) mean = np.mean(occupancies) if not standard_deviation < 0.4 * mean: streak_pixels.append(pixels) #all_streaks = all_streaks|set(pixels) if len(streak_pixels) == 0: return False if len(streak_pixels) > 1: completed = False i = 0 while not completed: same_object = False for pixel1 in streak_pixels[i]: if same_object: streak_pixels[i + 1] = streak_pixels[i + 1] + streak_pixels[i] streak_pixels.pop(i) i -= 1 break for pixel2 in streak_pixels[i + 1]: arr = pixel1.split(" ") y1 = int(arr[0]) x1 = int(arr[1]) arr = pixel2.split(" ") y2 = int(arr[0]) x2 = int(arr[1]) dist = ((x2 - x1)**2 + (y2 - y1)**2)**0.5 if dist < 100: same_object = True break i += 1 if i == len(streak_pixels) - 1: completed = True for pixels in streak_pixels: x_centre = 0 y_centre = 0 for i in range(len(pixels)): arr = pixels[i].split(" ") y = int(arr[0]) x = int(arr[1]) x_centre += x y_centre += y x_centre = x_centre / len(pixels) y_centre = y_centre / len(pixels) streaks.append([x_centre, y_centre]) self.streaks[n - 1] = streaks # ============================================================================= # for string in all_streaks: # # arr = string.split(" ") # y = int(arr[0]) # x = int(arr[1]) # # self.second_image[y][x] = 30000 # # path = self.directory + Constants.working_directory + Constants.image_directory + Constants.reduced_prefix # # image = Utilities.get_image_data(path, 189) # # set_n, i = Utilities.n_to_set_and_n(189) # # file = path + Constants.file_name + "_" + str(set_n) + "_" + Utilities.format_index(i) + Constants.fits_extension # head=getheader(file ,ignore_missing_end=True) # # hdu = PrimaryHDU(self.second_image, head) # hdul = HDUList([hdu], None) # hdul.writeto(self.directory + Constants.working_directory + "testimage.fits", overwrite=True) # # ============================================================================= if not len(streaks) == 0: return True return False
def dispense_samples(args, labware_dict, sample_data_dict, sample_parameters, left_pipette, right_pipette, water_aspirated): """ Dilute and dispense samples @param args: @param labware_dict: @param sample_data_dict: @param sample_parameters: @param left_pipette: @param right_pipette: @param water_aspirated: """ bottom_offset = float(args.BottomOffset) dilution_labware = labware_dict[args.DilutionPlateSlot] sample_destination_labware = labware_dict[args.PCR_PlateSlot] reagent_labware = labware_dict[args.ReagentSlot] water_res_well_dia = reagent_labware[args.WaterWell].diameter cone_vol = Utilities.labware_cone_volume(args, reagent_labware) water_tip_height = Utilities.res_tip_height( float(args.WaterResVol) - water_aspirated, water_res_well_dia, cone_vol, bottom_offset) dilution_plate_layout = Utilities.plate_layout() dilution_well_index = 0 for sample_key in sample_parameters: sample_source_labware = labware_dict[sample_parameters[sample_key][0]] sample_source_well = sample_parameters[sample_key][1] sample_dest_wells = sample_data_dict[sample_key][3] sample_vol = sample_data_dict[sample_key][0] diluent_vol = sample_data_dict[sample_key][1] diluted_sample_vol = sample_data_dict[sample_key][2] # If no dilution is necessary, dispense sample and continue if diluted_sample_vol == 0: sample_pipette, sample_loop, sample_vol = \ Utilities.pipette_selection(left_pipette, right_pipette, sample_vol) for well in sample_dest_wells: Utilities.dispensing_loop( args, sample_loop, sample_pipette, sample_source_labware[sample_source_well], sample_destination_labware[well], sample_vol, NewTip=True, MixReaction=True, touch=True) continue # Adjust volume of diluted sample to make sure there is enough diluted_template_needed = round( diluted_sample_vol * (len(sample_dest_wells) + 1.5), 2) diluted_template_factor = round( diluted_template_needed / (sample_vol + diluent_vol), 2) ''' diluted_template_on_hand = sample_vol+diluent_vol diluted_template_factor = 1.0 if diluted_template_needed <= diluted_template_on_hand: diluted_template_factor = diluted_template_needed/diluted_template_on_hand if diluted_template_factor <= 1.5 and (sample_vol * diluted_template_factor) < 10: diluted_template_factor = 2.0 ''' adjusted_sample_vol = round((sample_vol * diluted_template_factor), 1) diluent_vol = round((diluent_vol * diluted_template_factor), 1) # Reset the pipettes for the new volumes diluent_pipette, diluent_loop, diluent_vol = \ Utilities.pipette_selection(left_pipette, right_pipette, diluent_vol) sample_pipette, sample_loop, sample_vol = \ Utilities.pipette_selection(left_pipette, right_pipette, adjusted_sample_vol) # Make dilution, diluent first dilution_well = dilution_plate_layout[dilution_well_index] Utilities.dispensing_loop( args, diluent_loop, diluent_pipette, reagent_labware[args.WaterWell].bottom(water_tip_height), dilution_labware[dilution_well], diluent_vol, NewTip=True, MixReaction=False, touch=True) Utilities.dispensing_loop(args, sample_loop, sample_pipette, sample_source_labware[sample_source_well], dilution_labware[dilution_well], sample_vol, NewTip=True, MixReaction=True) water_aspirated += diluent_vol dilution_well_index += 1 water_tip_height = \ Utilities.res_tip_height(float(args.WaterResVol)-water_aspirated, water_res_well_dia, cone_vol, bottom_offset) # Add diluted sample to PCR plate for well in sample_dest_wells: sample_pipette, sample_loop, diluted_sample_vol = \ Utilities.pipette_selection(left_pipette, right_pipette, diluted_sample_vol) Utilities.dispensing_loop( args, sample_loop, sample_pipette, dilution_labware[dilution_well].bottom(bottom_offset), sample_destination_labware[well], diluted_sample_vol, NewTip=True, MixReaction=True) if sample_pipette.has_tip: sample_pipette.drop_tip() return water_aspirated
def run(ctx: protocol_api.ProtocolContext): ctx.comment("Begin {}".format(metadata['protocolName'])) # Turn on rail lights and pause program so user can load robot deck. # ctx.set_rail_lights(True) # ctx.pause("Load Labware onto robot deck and click resume when ready to continue") # ctx.home() ctx.set_rail_lights(False) # TSV file location on OT-2 tsv_file_path = "{0}var{0}lib{0}jupyter{0}notebooks{0}ProcedureFile.tsv".format( os.sep) # If not on the OT-2, get temp TSV file location on Win10 Computers for simulation if not os.path.isfile(tsv_file_path): tsv_file_path = "C:{0}Users{0}{1}{0}Documents{0}TempTSV.tsv".format( os.sep, os.getlogin()) sample_parameters, args = Utilities.parse_sample_template(tsv_file_path) labware_dict, left_tiprack_list, right_tiprack_list = Utilities.labware_parsing( args, ctx) # Pipettes left_pipette = ctx.load_instrument(args.LeftPipette, 'left', tip_racks=left_tiprack_list) right_pipette = ctx.load_instrument(args.RightPipette, 'right', tip_racks=right_tiprack_list) # Set the location of the first tip in box. with suppress(IndexError): left_pipette.starting_tip = left_tiprack_list[0].wells_by_name()[ args.LeftPipetteFirstTip] with suppress(IndexError): right_pipette.starting_tip = right_tiprack_list[0].wells_by_name()[ args.RightPipetteFirstTip] sample_data_dict, water_well_dict, target_well_dict, used_wells, layout_data, max_template_vol = \ sample_processing(args, sample_parameters) # This will output a plate layout file. Only does it during the simulation from our GUI if ctx.is_simulating() and platform.system() == "Windows": run_date = datetime.datetime.today().strftime("%a %b %d %H:%M %Y") plate_layout_string = \ "## ddPCR Setup\n## Setup Date:\t{}\n## Template User:\t{}\n" \ "# Format:\tTemplate | Target | Template Dilution | Template Volume in Reaction\n\n\t"\ .format(run_date, args.User) for i in range(12): plate_layout_string += "{}\t".format(i + 1) # I have to import this here because I have been unable to get natsort on the robot. import natsort for well in natsort.natsorted(layout_data): well_string = "\t".join(layout_data[well]) plate_layout_string += "\n{}\t{}\t".format(well, well_string) plate_layout_file = \ open("C:{0}Users{0}{1}{0}Documents{0}ddPCR_PlateLayout.tsv".format(os.sep, os.getlogin()), 'w') plate_layout_file.write(plate_layout_string) plate_layout_file.close() # Now do the actual dispensing. water_aspirated = dispense_water(args, labware_dict, water_well_dict, left_pipette, right_pipette) dispense_reagent_mix(args, labware_dict, target_well_dict, left_pipette, right_pipette) water_aspirated = dispense_samples(args, labware_dict, sample_data_dict, sample_parameters, left_pipette, right_pipette, water_aspirated) fill_empty_wells(args, used_wells, water_aspirated, labware_dict, left_pipette, right_pipette) ctx.comment("\nProgram Complete") if not ctx.is_simulating(): os.remove(tsv_file_path)
def gpu_ifs_transform(transformation=constants.ifs_fractals["fern"], width=600, height=600, num_points=100000, block_size=64, output_file="gpuOut.png"): """ This function will perform the Iterated Function System (IFS) fractal algorithm via CUDA. :param block_size: GPU Block Size :param transformation: A transformation matrix with 7 columns representing [a, b, c, d, e, f, prob] for the IFS function x_(n+1) = ax_n + by_n + e and y_(n+1) = cx_n + dy_n + f :param width: Width of the image in pixels :param height: Height of the image in pixels :param num_points: Number of points in fractal :param output_file: File to save the image to :return: algorithm runtime in seconds, number of points """ start = timer() # Generate Hammersley sequence block = (block_size, 1, 1) gpu_x = gpuarray.to_gpu(np.zeros(num_points, np.float32)) gpu_y = gpuarray.to_gpu(np.zeros(num_points, np.float32)) data_generation = SourceModule(KernelCode.gpu_hammersley_kernel_code) hammersley_func = data_generation.get_function("hammersley") hammersley_func(np.int32(num_points), gpu_x, gpu_y, block=block) transformation = np.array(transformation, np.float32) gpu_transform = gpuarray.to_gpu(transformation) rows, cols = transformation.shape grid = (num_points, 1, 1) mod = SourceModule(KernelCode.ifs_transform_kernel_code, no_extern_c=True) ifs_func = mod.get_function("phase1Transform") ifs_func(gpu_x, gpu_y, gpu_transform, np.int32(num_points), np.int32(rows), block=block, grid=grid, shared=sys.getsizeof(gpu_transform)) curr_iter = 0 while curr_iter < 15: ifs_func(gpu_x, gpu_y, gpu_transform, np.int32(num_points), np.int32(rows), block=block, grid=grid, shared=sys.getsizeof(gpu_transform)) curr_iter += 1 x = gpu_x.get() y = gpu_y.get() points = list(zip(x, y)) run_time = timer() - start Utilities.draw_image(points, width, height, output_file) return run_time, len(points)
def MakeImageDocument(url, imageType, exportInfo): # First make a bitmap context for a US Letter size # raster at the requested resolution. dpi = exportInfo.dpi width = int(8.5 * dpi) height = int(11 * dpi) # For JPEG output type the bitmap should not be transparent. If other types are added that # do not support transparency, this code should be updated to check for those types as well. needTransparentBitmap = imageType.lower( ) != LaunchServices.kUTTypeJPEG.lower() # Create an RGB Bitmap context using the generic calibrated RGB color space # instead of the display color space. useDisplayColorSpace = False c = createRGBBitmapContext(width, height, useDisplayColorSpace, needTransparentBitmap) if c is None: print("Couldn't make destination bitmap context") return -1 # Scale the coordinate system based on the resolution in dots per inch. Quartz.CGContextScaleCTM(c, dpi / 72, dpi / 72) # Set the font smoothing parameter to false since it's better to # draw any text without special LCD text rendering when creating # rendered data for export. if hasattr(Quartz, "CGContextSetShouldSmoothFonts"): Quartz.CGContextSetShouldSmoothFonts(c, False) # Set the scaling factor for shadows. This is a hack so that # drawing code that needs to know the scaling factor can # obtain it. Better would be that DispatchDrawing and the code # it calls would take this scaling factor as a parameter. Utilities.setScalingFactor(dpi / 72) # Draw into that raster... AppDrawing.DispatchDrawing(c, exportInfo.command) # Set the scaling factor back to 1.0. Utilities.setScalingFactor(1.0) # Create an image from the raster data. Calling # createImageFromBitmapContext gives up ownership # of the raster data used by the context. image = createImageFromBitmapContext(c) # Release the context now that the image is created. del c if image is None: # Users of this code should update this to be an error code they find useful. return -1 # Now export the image. if exportInfo.useQTForExport: exportCGImageToFileWithQT(image, url, imageType, exportInfo.dpi) else: exportCGImageToFileWithDestination(image, url, imageType, exportInfo.dpi)
def sample_processing(args, sample_parameters): sample_data_dict = defaultdict(list) target_well_dict = defaultdict(list) water_well_dict = defaultdict(float) layout_data = defaultdict(list) # Builds the data frame for printing the plate layout file for k in ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H']: layout_data[k] = [ '', '', '', '', '', '', '', '', '', '', '', '', ] plate_layout_by_column = Utilities.plate_layout() used_wells = [] dest_well_count = 0 target_list = [] for sample_key in sample_parameters: sample_name = sample_parameters[sample_key][2] sample_string = sample_name sample_concentration = float(sample_parameters[sample_key][3]) targets = sample_parameters[sample_key][4].split(",") replicates = int(sample_parameters[sample_key][5]) sample_vol, diluent_vol, diluted_sample_vol, reaction_water_vol, max_template_vol = \ Utilities.calculate_volumes(args, sample_concentration) sample_wells = [] for target in targets: target_list.append(target) target_name = getattr(args, "Target_{}".format(target))[1] for i in range(replicates): well = plate_layout_by_column[dest_well_count] row = well[0] column = int(well[1:]) - 1 s_volume = diluted_sample_vol if diluent_vol == 0: dilution = "Neat" s_volume = sample_vol else: dilution = "1:{}".format( int((sample_vol + diluent_vol) / sample_vol)) layout_data[row][column] = "{}|{}|{}|{}"\ .format(sample_string, target_name, dilution, s_volume) water_well_dict[well] = reaction_water_vol target_well_dict[target].append(well) sample_wells.append(well) used_wells.append(well) dest_well_count += 1 sample_data_dict[sample_key] = \ [round(sample_vol, 1), round(diluent_vol, 1), round(diluted_sample_vol, 1), sample_wells] # Define our no template control wells for the targets. for target in target_well_dict: target_list.append(target) target_list = list(set(target_list)) for target in target_list: control_name = "Water" target_data = getattr(args, "Target_{}".format(target)) target_name = target_data[1] well = plate_layout_by_column[dest_well_count] used_wells.append(well) row = well[0] column = int(well[1:]) - 1 layout_data[row][column] = "{}|{}|NA|{}".format( control_name, target_name, max_template_vol) water_well_dict[well] = max_template_vol dest_well_count += 1 target_well_dict[target].append(well) return sample_data_dict, water_well_dict, target_well_dict, used_wells, layout_data, max_template_vol
# how many samples per batch to load batch_size = 20 # show image sample before training showImages = True # create loss logger file names validationLossFile = architecture+'/validationLoss.csv' trainingLossFile = architecture+'/trainingLoss.csv' # ============================================================================= # Setup and user feedback # ============================================================================= # try to import the model architecture definition module cnn = util.importModelArchitecture(package, architecture) # check if CUDA is available on this computer train_on_gpu = torch.cuda.is_available() print('Cuda available?: ' + ('Yes' if train_on_gpu else 'No')) # ============================================================================= # Obtaining and preprocessing data # ============================================================================= ''' Data Augmentation Defines a basic transformation pipeline for data augmentation. Transformations may include random (horizontal) flips of pictures or small roatations.
default="../../All_Input/input_xml/norm_factor_EFT_1D.txt") parser.add_argument('--syst', default="../../All_Input/input_xml/systematics_fake.txt") parser.add_argument('--jetptcut', default="35") parser.add_argument('--discriminant', default="BDT_1bin") parser.add_argument('--outpath', '-o', default="../../All_Input/workspace_xml/") parser.add_argument('--Asimov', action="store_true") parser.add_argument('--dtd', default="../../All_Input/input_xml/HistFactorySchema.dtd") parser.add_argument('--wspath', default="../../All_Input/workspace/") parser.add_argument('--mode', default="vv") args = parser.parse_args() ROOT.gROOT.SetBatch(ROOT.kTRUE) gSystem.Load("../macros/writeXML/writeXML_cpp.so") Utilities.try_makedir(join(args.outpath, args.discriminant)) Utilities.try_makedir(join(args.wspath, args.discriminant)) #ROOT.H4l_XML.writeXML(ErrorType = "THEO SYS EXP", sample_in = XML_PATH + SAMPLE_IN, category_in = XML_PATH + CATEGORY_IN, norm_in = XML_PATH + NORM_IN, syst_in = XML_PATH + SYST_IN, jetptcut = "35", discriminant = "BDT", IsAsimov = true) ROOT.H4l_XML.writeXML(args.err, args.sample, args.category, args.norm, args.syst, args.jetptcut, args.discriminant, args.outpath, args.Asimov, args.mode) ROOT.H4l_XML.writeDriver(args.dtd, args.wspath, args.category, args.outpath, args.discriminant) os.system("hist2workspace -standard_form {0}".format( join(args.outpath, args.discriminant, "driver.xml")))
import genetic_algorithm as GA import Utilities as TOOLS import Networks import tensorflow as tf from keras.datasets import cifar10 from keras.utils.np_utils import to_categorical import matplotlib.pyplot as plt from datetime import date from operator import itemgetter import os #Global Variables GENE_SET = '10' #Binary values for model chomosome Constant DO NOT CHANGE MAX_NUM_NODES = int(TOOLS.max_bi(len(TOOLS.int_to_bit(50))), 2) #Max number of nodes in each layer MAX_NUM_HIDDEN_LAYERS = int(TOOLS.max_bi(len(TOOLS.int_to_bit(10))), 2) #Max number of hidden layers TOOLS.global_variables( ['sigmoid', 'tanh', 'relu', 'elu'], #Activation Function [.1, .01, .001, .0001, .00001, .000001, .0000001, .00000001 ], #Learning Rate [0, .1, .2, .3, .4, .5, .6, .7]) #Dropout Rate #Change for increased performance POPULATION_SIZE = 10 #zeo index MUTATION_CHANCE = 0.2 #Chance of randomly tweaking the models chromosome CROSSOVER_CHANCE = 0.45 #Chance for uniform crossover between two parents to occure KEEP = 0.4 #Keep the top X percent of the population maxGen = 10 #Number of generations
def main(): """ Main entry point into the application. """ import getopt try: opts, args = getopt.getopt(sys.argv[1:], "c:ehio:Rrt:Vx:", [ "exclude=", "extension=", "help", "noindex", "noempty", "outdir=", "recursive", "style-sheet=", "version", "exclude-file=", "eol=", "body-color=", "body-background-color=", "l1header-color=", "l1header-background-color=", "l2header-color=", "l2header-background-color=", "cfheader-color=", "cfheader-background-color=", "link-color=", "create-qhp", "qhp-outdir=", "qhp-namespace=", "qhp-virtualfolder=", "qhp-filtername=", "qhp-filterattribs=", "qhp-title=", "create-qhc", ]) except getopt.error: usage() excludeDirs = [ "CVS", ".svn", "_svn", ".ropeproject", "_ropeproject", ".eric6project", "_eric6project", "dist", "build", "doc", "docs" ] excludePatterns = [] outputDir = "doc" recursive = False doIndex = True noempty = False newline = None stylesheetFile = "" colors = eric6docDefaultColors.copy() qtHelpCreation = False qtHelpOutputDir = "help" qtHelpNamespace = "" qtHelpFolder = "source" qtHelpFilterName = "unknown" qtHelpFilterAttribs = "" qtHelpTitle = "" qtHelpCreateCollection = False for k, v in opts: if k in ["-o", "--outdir"]: outputDir = v elif k in ["-R", "-r", "--recursive"]: recursive = True elif k in ["-x", "--exclude"]: excludeDirs.append(v) elif k == "--exclude-file": excludePatterns.append(v) elif k in ["-i", "--noindex"]: doIndex = False elif k in ["-e", "--noempty"]: noempty = True elif k in ["-h", "--help"]: usage() elif k in ["-V", "--version"]: version() elif k in ["-c", "--style-sheet"]: stylesheetFile = v elif k in ["-t", "--extension"]: if not v.startswith("."): v = ".{0}".format(v) supportedExtensions.append(v) elif k == "--eol": if v.lower() == "cr": newline = '\r' elif v.lower() == "lf": newline = '\n' elif v.lower() == "crlf": newline = '\r\n' elif k == "--body-color": colors['BodyColor'] = v elif k == "--body-background-color": colors['BodyBgColor'] = v elif k == "--l1header-color": colors['Level1HeaderColor'] = v elif k == "--l1header-background-color": colors['Level1HeaderBgColor'] = v elif k == "--l2header-color": colors['Level2HeaderColor'] = v elif k == "--l2header-background-color": colors['Level2HeaderBgColor'] = v elif k == "--cfheader-color": colors['CFColor'] = v elif k == "--cfheader-background-color": colors['CFBgColor'] = v elif k == "--link-color": colors['LinkColor'] = v elif k == "--create-qhp": qtHelpCreation = True elif k == "--qhp-outdir": qtHelpOutputDir = v elif k == "--qhp-namespace": qtHelpNamespace = v elif k == "--qhp-virtualfolder": qtHelpFolder = v elif k == "--qhp-filtername": qtHelpFilterName = v elif k == "--qhp-filterattribs": qtHelpFilterAttribs = v elif k == "--qhp-title": qtHelpTitle = v elif k == "--create-qhc": qtHelpCreateCollection = True if not args: usage() if qtHelpCreation and \ (qtHelpNamespace == "" or qtHelpFolder == "" or '/' in qtHelpFolder or qtHelpTitle == ""): usage() if qtHelpCreation: from PyQt5.QtCore import QCoreApplication app = QCoreApplication(sys.argv) # __IGNORE_WARNING__ input = output = 0 basename = "" if outputDir: if not os.path.isdir(outputDir): try: os.makedirs(outputDir) except EnvironmentError: sys.stderr.write( "Could not create output directory {0}.".format(outputDir)) sys.exit(2) else: outputDir = os.getcwd() outputDir = os.path.abspath(outputDir) if stylesheetFile: try: sf = open(stylesheetFile, "r", encoding="utf-8") stylesheet = sf.read() sf.close() except IOError: sys.stderr.write( "The CSS stylesheet '{0}' does not exist\n".format( stylesheetFile)) sys.stderr.write("Disabling CSS usage.\n") stylesheet = None else: stylesheet = None indexGenerator = IndexGenerator(outputDir, colors, stylesheet) if qtHelpCreation: if qtHelpOutputDir: if not os.path.isdir(qtHelpOutputDir): try: os.makedirs(qtHelpOutputDir) except EnvironmentError: sys.stderr.write( "Could not create QtHelp output directory {0}.".format( qtHelpOutputDir)) sys.exit(2) else: qtHelpOutputDir = os.getcwd() qtHelpOutputDir = os.path.abspath(qtHelpOutputDir) qtHelpGenerator = QtHelpGenerator(outputDir, qtHelpOutputDir, qtHelpNamespace, qtHelpFolder, qtHelpFilterName, qtHelpFilterAttribs, qtHelpTitle, qtHelpCreateCollection) for arg in args: if os.path.isdir(arg): if os.path.exists( os.path.join(arg, Utilities.joinext("__init__", ".py"))): basename = os.path.dirname(arg) if arg == '.': sys.stderr.write("The directory '.' is a package.\n") sys.stderr.write( "Please repeat the call giving its real name.\n") sys.stderr.write("Ignoring the directory.\n") continue else: basename = arg if basename: basename = "{0}{1}".format(basename, os.sep) if recursive and not os.path.islink(arg): names = [arg] + Utilities.getDirs(arg, excludeDirs) else: names = [arg] else: basename = "" names = [arg] for filename in names: inpackage = False if os.path.isdir(filename): files = [] for ext in supportedExtensions: files.extend( glob.glob( os.path.join(filename, Utilities.joinext("*", ext)))) initFile = os.path.join(filename, Utilities.joinext("__init__", ext)) if initFile in files: inpackage = True files.remove(initFile) files.insert(0, initFile) else: if Utilities.isWindowsPlatform() and glob.has_magic(filename): files = glob.glob(filename) else: files = [filename] for file in files: skipIt = False for pattern in excludePatterns: if fnmatch.fnmatch(os.path.basename(file), pattern): skipIt = True break if skipIt: continue try: module = Utilities.ModuleParser.readModule( file, basename=basename, inpackage=inpackage, extensions=supportedExtensions) moduleDocument = ModuleDocument(module, colors, stylesheet) doc = moduleDocument.genDocument() except IOError as v: sys.stderr.write("{0} error: {1}\n".format(file, v[1])) continue except ImportError as v: sys.stderr.write("{0} error: {1}\n".format(file, v)) continue input = input + 1 f = Utilities.joinext( os.path.join(outputDir, moduleDocument.name()), ".html") # remember for index file generation indexGenerator.remember(file, moduleDocument, basename) # remember for QtHelp generation if qtHelpCreation: qtHelpGenerator.remember(file, moduleDocument, basename) if (noempty or file.endswith('__init__.py')) \ and moduleDocument.isEmpty(): continue # generate output try: out = open(f, "w", encoding="utf-8", newline=newline) out.write(doc) out.close() except IOError as v: sys.stderr.write("{0} error: {1}\n".format(file, v[1])) else: sys.stdout.write("{0} ok\n".format(f)) output = output + 1 sys.stdout.flush() sys.stderr.flush() # write index files if doIndex: indexGenerator.writeIndices(basename, newline=newline) # generate the QtHelp files if qtHelpCreation: qtHelpGenerator.generateFiles(newline=newline) sys.exit(0)
import os import google.cloud import sys import pandas as pd import numpy as np sys.path.append(os.path.abspath(__file__)) import Utilities as utils import Constants as c from google.cloud import bigquery from concurrent.futures import ThreadPoolExecutor logger = utils.get_logger() def create_directory(name): try: os.mkdir(name) except OSError as error: print(error) project_name = "angular/angular" dir_name = project_name.split('/')[1] outputDirectoryPath = "scripts/exports/{directory}".format(directory=dir_name) def get_PRs(project_name, task_name, version): header = [ c.PROJECT, c.VERSION, c.DATE, c.TASK, c.T_LINE, c.T_MODULE, c.NT, c.T_COMMITS, c.MODULE, c.LINE, c.CONTRIBUTIONS, c.CONTRIBUTORS, c.COMMENTS, c.COMMENTERS ]
def train(bit_model, input_shape, classes, x_train, x_test, y_train, y_test, batch_size): """ Train each model within the generation return [bit_model, acc, model] bit_model String of binary 1/0 for offspring generation Acc Accuracy of the model for fitness calculation Model Keras model for saving the json file """ optimizer = 'adam' #Can be optimized wi, x_train, x_test, y_train, y_testth GA cost = 'mse' epochs = 100 #Find out how to use the learning rate num_hidden_layers, num_nodes_per_layer, learning_rate, activation_function_per_layer, dropout_rate, used = TOOLS.ANN_bit_to_model( bit_model, MAX_NUM_HIDDEN_LAYERS, MAX_NUM_NODES, classes) network = Networks.Model() model = network.create_architecture(num_hidden_layers, num_nodes_per_layer, activation_function_per_layer, dropout_rate, input_shape, optimizer, cost) acc = network.train(model, x_train, x_test, y_train, y_test, batch_size, epochs) #print(model) #model.save_weights("model.h5") #print("Saved model to disk") #exit() return [bit_model, acc, model]
def __init__(self, args, poisonIndex, transform=None): self.fineTuneFile = open(args.dataSplitDirectory + "fineTuneFile.txt") self.poisonFile = open(args.dataSplitDirectory + args.architecture + "_Poison.txt") self.poisonIndex = poisonIndex self.transform = transform self.classBalance = copy.deepcopy(args.classBalance) self.examples = copy.deepcopy(args.classBalance) self.K = args.K self.replicateImbalance = args.replicateImbalance self.extractedFeatures = torch.load(args.featureDirectory + args.architecture + "_CIFAR10_Features.pth") self.imageFiles = {} self.balancedImages = [] self.filteredImages = [] self.convexPolytopePoison = [] self.addIndex = set() for line in self.poisonFile: imgLocation, ID = line.split() index = imgLocation.split("/")[-1] ID = int(ID.strip("\n")) if ID not in self.imageFiles.keys(): self.imageFiles[ID] = [] if args.poisonImageDirectory + str(self.poisonIndex) in imgLocation: self.addIndex.add(index) self.imageFiles[ID].append((imgLocation, ID)) self.classBalance[ID] = self.classBalance[ID] - 1 self.convexPolytopePoison.append(imgLocation) for line in self.fineTuneFile: imgLocation, ID = line.split() index = imgLocation.split("/")[-1] ID = int(ID.strip("\n")) if ID not in self.imageFiles.keys(): self.imageFiles[ID] = [] if index not in self.addIndex and self.classBalance[ID] > 0: self.addIndex.add(index) self.imageFiles[ID].append((imgLocation, ID)) self.classBalance[ID] = self.classBalance[ID] - 1 if self.replicateImbalance: maxClass = max(self.examples) classWeight = [] for i in self.examples: classWeight.append(math.ceil(maxClass / i)) for key in self.imageFiles: self.balancedImages = self.balancedImages + (classWeight[key] * self.imageFiles[key])[0:maxClass] else: for key in self.imageFiles: self.balancedImages = self.balancedImages + self.imageFiles[key] KNN = KNeighborsClassifier(algorithm='brute', n_neighbors=self.K) trainFeatures = [] trainLabels = [] for data in self.balancedImages: imgLocation, ID = data FV = self.extractedFeatures[imgLocation].cpu().numpy() trainFeatures.append(FV) trainLabels.append(ID) KNN.fit(trainFeatures, trainLabels) KNNLabels = KNN.predict(trainFeatures) cleanImages = np.equal(KNNLabels, trainLabels) TP, FP, TN, FN = 0, 0, 0, 0 for data, valid in zip(self.balancedImages, cleanImages): imgLocation, ID = data if valid: self.filteredImages.append((imgLocation, ID)) if imgLocation not in self.convexPolytopePoison: TP = TP + 1 else: FP = FP + 1 else: if imgLocation in self.convexPolytopePoison: TN = TN + 1 else: FN = FN + 1 try: MCC = ((TP * TN) - (FP * FN)) / np.sqrt((TP + FP) * (TP + FN) * (TN + FP) * (TN + FN)) except: MCC = None print("True Positive: " + str(TP) + " | " + "True Negative: " + str(TN) + " | " + "False Positive: " + str(FP) + " | " + "False Negative: " + str(FN)) print("Matthews Correlation Coefficient: " + str(MCC)) Utilities.writeLog(args.logFileLocation, "True Positive: " + str(TP) + " | " + "True Negative: " + str(TN) + " | " + "False Positive: " + str(FP) + " | " + "False Negative: " + str(FN)) Utilities.writeLog(args.logFileLocation, "Matthews Correlation Coefficient: " + str(MCC))
model = SENet18() if torch.cuda.device_count() > 1: model = nn.DataParallel(model) CheckPoint = torch.load(checkPointDirectory + "/" + modelName + ".pth") model.load_state_dict(CheckPoint["net"]) model.to(device) files = ["./CIFAR10/DataSplit/trainFile.txt", "./CIFAR10/DataSplit/testFile.txt", "./CIFAR10/DataSplit/fineTuneFile.txt", "./CIFAR10/DataSplit/" + modelName + "_Poison.txt"] extractedFeatures = {} for testFile in files: CIFAR = dataLoader.ConvexPolytopeFeatureExtraction_DataLoader(testFile, validationAugmentation) testData = DataLoader(CIFAR, batch_size=BATCHSIZE, shuffle=False, num_workers=WORKERS) featureVector, ID, imgLocation = Utilities.featureExtraction(model, device, testData) for data in zip(featureVector, imgLocation): FV, IL = data extractedFeatures[IL] = FV torch.save(extractedFeatures, "./CIFAR10/Features/" + modelName + "_CIFAR10_Features.pth")
global UPConfig UPConfig = UPConfig.ReadUPConfigFromGDB("..//testing","calaveras_complete.gdb") TimeStep = UPConfig['TimeSteps'][0] # Logger("Testing Making Allocation Table") # allocTable = MakeAllocTables(UPConfig) # Logger("Testing Saving Allocation Table") # try: # Utilities.SavePdDataFrameToTable(allocTable, "..//testing/Calaveras.gdb", 'z_testTable') # except Exception, e: # Logger("Saving Allocation Table Failed: {err}".format(err=str(e))) Logger("Testing DevSpace Calculations") devSpace = MakeDevSpace(UPConfig, TimeStep) Utilities.SavePdDataFrameToTable(devSpace, "..//testing/Calaveras.gdb", "z_devspace") Logger("DevSpace Calculation Complete") Logger("Testing Is Higher Priority") print(IsHigherPriority(UPConfig,'rm',['rh','rm','rl'],[])) # should return True print(IsHigherPriority(UPConfig,'rm',['rh','rm','rl'],['rl'])) # should return True print(IsHigherPriority(UPConfig,'rm',['rh','rm','rl'],['rh'])) # should return False print(IsHigherPriority(UPConfig,'rh',['rh','rm','rl'],['rh'])) # should return True Logger("Testing Is Higher Priority Complete") # Logger("Test Add LU Field") # AddLUField(UPConfig,"upo_cum_alloc_ts1")
def main(N, timesteps, agent, params, dt, det=False, r_0=np.zeros(2), der_0=np.zeros(2), sc_0=np.identity(2), desc_0=0 * np.identity(2), bench=-0.1, bench_noise=0.3): a, dea, d, b, e = Tools.Matrices_Calculator( 'Fisher', params) #calculation of 2x2 matrices A, deA, D, B, E = matrix_parallel_Fisher( N, a, dea, d, b, e) #calculation of 10x2x2 matrices SY = symplectic(N) #10x2x2 symplectic form R, deR, SC, deSC = initial_conditions(N, r_0, der_0, sc_0, desc_0) #fix initial conditions R2, deR2, SC2, deSC2 = R, deR, SC, deSC #fix initial conditions for dumb action dW = np.random.randn(N, 2) * ( dt**0.5 ) #extraction of wiener increments (the same for both network and dumb feedback) dedW = (2**0.5) * prod_v((np.transpose(B, axes=(0, 2, 1))), (deR)) * dt #dedw CURR = -(2**0.5) * prod_v((np.transpose(B, axes=(0, 2, 1))), (R)) * dt + dW #current calculation dedW2 = (2**0.5) * prod_v((np.transpose(B, axes=(0, 2, 1))), (deR2)) * dt #dedw for dumb action CURR2 = -(2**0.5) * prod_v((np.transpose(B, axes=(0, 2, 1))), (R2)) * dt + dW #current for dumb action Fisher_Cl = 0 #initialize fisher for agent Fisher_Cl2 = 0 #initialize fisher for dumb action INFOS = [] #empty list for dicts of infos on agent INFOS2 = [] #empty dict for dicts of infos on dumb action obs = obs_prep( R, deR, CURR, SC, deSC, N) #prepare observation to match observation shape of the model #evolution and information collection for all of trajectories at the same time for i in range(0, timesteps): if i % int(timesteps * 1e-1) == 0: percentage = (i / (timesteps * 1e-2)) print('{}%'.format(percentage)) action, _tmp = agent.predict(obs, deterministic=det) action = np.array(action[:, 0]) J = action[:, np.newaxis, np.newaxis] * SY R, SC, deR, deSC = fishsystem_nocost_RK4(R, deR, SC, deSC, A, deA, D, B, E, dt, dW, dedW, J) dFisher_Cl = dClassical_Fisher(deR, B, dt) Fisher_Cl = Fisher_Cl + dFisher_Cl Fisher_Quantum = Quantum_Fisher(N, SC, deSC, deR) action2 = benchmark(N, bench, bench_noise) J2 = action2[:, np.newaxis, np.newaxis] * SY R2, SC2, deR2, deSC2 = fishsystem_nocost_RK4(R2, deR2, SC2, deSC2, A, deA, D, B, E, dt, dW, dedW2, J2) dFisher_Cl2 = dClassical_Fisher(deR2, B, dt) Fisher_Cl2 = Fisher_Cl2 + dFisher_Cl2 Fisher_Quantum2 = Quantum_Fisher(N, SC2, deSC2, deR2) #data collection R_mean = np.mean(R, axis=0) deR_mean = np.mean(deR, axis=0) SC_mean = np.mean(SC, axis=0) deSC_mean = np.mean(deSC, axis=0) Fisher_Cl_mean = np.mean(Fisher_Cl, axis=0) Fisher_Quantum_mean = np.mean(Fisher_Quantum, axis=0) action_mean = np.mean(action, axis=0) R_std = np.std(R, axis=0) deR_std = np.std(deR, axis=0) SC_std = np.std(SC, axis=0) deSC_std = np.std(deSC, axis=0) Fisher_Cl_std = np.std(Fisher_Cl, axis=0) Fisher_Quantum_std = np.std(Fisher_Quantum, axis=0) action_std = np.std(action, axis=0) R2_mean = np.mean(R2, axis=0) deR2_mean = np.mean(deR2, axis=0) SC2_mean = np.mean(SC2, axis=0) deSC2_mean = np.mean(deSC2, axis=0) Fisher_Cl2_mean = np.mean(Fisher_Cl2, axis=0) Fisher_Quantum2_mean = np.mean(Fisher_Quantum2, axis=0) action2_mean = np.mean(action2, axis=0) R2_std = np.std(R2, axis=0) deR2_std = np.std(deR2, axis=0) SC2_std = np.std(SC2, axis=0) deSC2_std = np.std(deSC2, axis=0) Fisher_Cl2_std = np.std(Fisher_Cl2, axis=0) Fisher_Quantum2_std = np.std(Fisher_Quantum2, axis=0) action2_std = np.std(action2, axis=0) infos = {} #empty dict for infos on agent infos2 = {} #empty dict for infos on dumb action infos['r_0 mean'] = R_mean[0] infos['r_1 mean'] = R_mean[1] infos['der_0 mean'] = deR_mean[0] infos['der_1 mean'] = deR_mean[1] infos['sc_00 mean'] = SC_mean[0, 0] infos['sc_01 mean'] = SC_mean[0, 1] infos['sc_10 mean'] = SC_mean[1, 0] infos['sc_11 mean'] = SC_mean[1, 1] infos['desc_00 mean'] = deSC_mean[0, 0] infos['desc_01 mean'] = deSC_mean[0, 1] infos['desc_10 mean'] = deSC_mean[1, 0] infos['desc_11 mean'] = deSC_mean[1, 1] infos['Fisher_Cl mean'] = Fisher_Cl_mean infos['Fisher_Quantum mean'] = Fisher_Quantum_mean infos['action mean'] = action_mean infos['r_0 std'] = R_std[0] infos['r_1 std'] = R_std[1] infos['der_0 std'] = deR_std[0] infos['der_1 std'] = deR_std[1] infos['sc_00 std'] = SC_std[0, 0] infos['sc_01 std'] = SC_std[0, 1] infos['sc_10 std'] = SC_std[1, 0] infos['sc_11 std'] = SC_std[1, 1] infos['desc_00 std'] = deSC_std[0, 0] infos['desc_01 std'] = deSC_std[0, 1] infos['desc_10 std'] = deSC_std[1, 0] infos['desc_11 std'] = deSC_std[1, 1] infos['Fisher_Cl std'] = Fisher_Cl_std infos['Fisher_Quantum std'] = Fisher_Quantum_std infos['action std'] = action_std INFOS.append(infos) infos2['r_0 mean'] = R2_mean[0] infos2['r_1 mean'] = R2_mean[1] infos2['der_0 mean'] = deR2_mean[0] infos2['der_1 mean'] = deR2_mean[1] infos2['sc_00 mean'] = SC2_mean[0, 0] infos2['sc_01 mean'] = SC2_mean[0, 1] infos2['sc_10 mean'] = SC2_mean[1, 0] infos2['sc_11 mean'] = SC2_mean[1, 1] infos2['desc_00 mean'] = deSC2_mean[0, 0] infos2['desc_01 mean'] = deSC2_mean[0, 1] infos2['desc_10 mean'] = deSC2_mean[1, 0] infos2['desc_11 mean'] = deSC2_mean[1, 1] infos2['Fisher_Cl mean'] = Fisher_Cl2_mean infos2['Fisher_Quantum mean'] = Fisher_Quantum2_mean infos2['action mean'] = action2_mean infos2['r_0 std'] = R2_std[0] infos2['r_1 std'] = R2_std[1] infos2['der_0 std'] = deR2_std[0] infos2['der_1 std'] = deR2_std[1] infos2['sc_00 std'] = SC2_std[0, 0] infos2['sc_01 std'] = SC2_std[0, 1] infos2['sc_10 std'] = SC2_std[1, 0] infos2['sc_11 std'] = SC2_std[1, 1] infos2['desc_00 std'] = deSC2_std[0, 0] infos2['desc_01 std'] = deSC2_std[0, 1] infos2['desc_10 std'] = deSC2_std[1, 0] infos2['desc_11 std'] = deSC2_std[1, 1] infos2['Fisher_Cl std'] = Fisher_Cl2_std infos2['Fisher_Quantum std'] = Fisher_Quantum2_std infos2['action std'] = action2_std INFOS2.append(infos2) dW = np.random.randn(N, 2) * (dt**0.5) dedW = (2**0.5) * prod_v((np.transpose(B, axes=(0, 2, 1))), (deR)) * dt CURR = -(2**0.5) * prod_v((np.transpose(B, axes=(0, 2, 1))), (R)) * dt + dW dedW2 = (2**0.5) * prod_v((np.transpose(B, axes=(0, 2, 1))), (deR2)) * dt CURR2 = -(2**0.5) * prod_v((np.transpose(B, axes=(0, 2, 1))), (R2)) * dt + dW obs = obs_prep(R, deR, CURR, SC, deSC, N) #save in dataframes DATA = pd.DataFrame(INFOS) DATA2 = pd.DataFrame(INFOS2) return DATA, DATA2
plt.clf() plt.plot(epochs, val_loss, 'r', label='Validation Loss') plt.plot(epochs, train_loss, 'b', label='Train Loss') plt.title('Loss / Mean Sqared Error') plt.xlabel('Epochs') plt.ylabel('Accuracy') plt.legend() plt.savefig('model_phase01_loss.png') if __name__ == "__main__": script_start = datetime.datetime.now() #minMaxValues = Utilities.readMinMaxFromCSV(minMaxCSVpath) [images, categories, filenames] = Utilities.loadImagesAndCategories(images, imgsDir, categories, normalizedDataPath, phase = 1, inputWidth = inputWidth, inputHeight = inputHeight) [testImages, testLabels] = trainTestDatasetSplit(images, categories) model_name = "model_phase01.h5" model = cnn.create_model(inputWidth, inputHeight, 1, outputNo) # change to cnn input format df_im = np.asarray(images) df_im = df_im.reshape(df_im.shape[0], inputWidth, inputHeight, 1) df_cat = np.asarray(categories) df_cat = df_cat.reshape(df_cat.shape[0], outputNo) tr_im, val_im, tr_cat, val_cat = train_test_split(df_im, df_cat, test_size=0.2) tensorboard = TensorBoard(log_dir=imgsDir + "logs_img1" + "\{}".format(time()))
def MakeDevSpace(UPConfig,TimeStep): """ Make the devSpaceTable to contain the list of unconstrained space for each land use in each polygon. This accounts for the effects of all constraints. It also contains the general plan class for each polygon. The last step fills in all No Data values with 0 (assuming that if there is no data, there is also no available space). Called By: Allocation.PriAlloc Calls: MakePolyList MakeGPList Utilities.MergeDataFrames Arguments: UPConfig TimeStep (as list) - ['ts1','timestep1'] Returns: devSpaceTable """ # Build DevSpaceTable for this time step Logger("Preparing Developable Space Table") devSpaceTable = MakePolyList(UPConfig) for lu in UPConfig['LUPriority']: # get unconstrained space dswhereclause = """ TimeStep = '{ts}' and lu = '{lu}' """.format(ts= TimeStep[0],lu=lu) dsarray = arcpy.da.TableToNumPyArray(os.path.join(UPConfig['paths']['dbpath'],UPConfig['paths']['dbname'],'up_const'),[UPConfig['BaseGeom_id'],'developable_acres'],dswhereclause) # TODO: rename this to unconstrained_acres dsdf = pd.DataFrame(dsarray) dsdf.columns = [[UPConfig['BaseGeom_id'],'uncon_ac_{ts}_{lu}'.format(ts=TimeStep[0],lu=lu)]] dsarray = None # get gp permissablity (boolean) gparray = arcpy.da.TableToNumPyArray(os.path.join(UPConfig['paths']['dbpath'],UPConfig['paths']['dbname'],'up_bg_gp_avail_{ts}'.format(ts=TimeStep[0])),[UPConfig['BaseGeom_id'],'gp_{lu}'.format(lu=lu)]) gpdf = pd.DataFrame(gparray) gpdf.columns = [[UPConfig['BaseGeom_id'],'gp_{ts}_{lu}'.format(ts=TimeStep[0],lu=lu)]] gparray = None # get net weights wtwhereclause = """ timestep = '{ts}' and lu = '{lu}' """.format(ts= TimeStep[0],lu=lu) wtarray = arcpy.da.TableToNumPyArray(os.path.join(UPConfig['paths']['dbpath'],UPConfig['paths']['dbname'],'up_net_weights'),[UPConfig['BaseGeom_id'],'weight'],wtwhereclause) wtdf = pd.DataFrame(wtarray) wtdf.columns = [[UPConfig['BaseGeom_id'],'wt_{ts}_{lu}'.format(ts=TimeStep[0],lu=lu)]] wtarray = None # #for debug # if lu == 'RH': # df1 = Utilities.ConvertPdDataFrameToNumpyArray(devSpaceTable) # df2 = Utilities.ConvertPdDataFrameToNumpyArray(gpdf) # df3 = Utilities.ConvertPdDataFrameToNumpyArray(dsdf) # df4 = Utilities.ConvertPdDataFrameToNumpyArray(wtdf) # arcpy.da.NumPyArrayToTable(devSpaceTable,r"G:\Public\UPLAN\Calaveras\Debug\devSpaceTable.csv") # arcpy.da.NumPyArrayToTable(gpdf,r"G:\Public\UPLAN\Calaveras\Debug\gpdf.csv") # arcpy.da.NumPyArrayToTable(dsdf,r"G:\Public\UPLAN\Calaveras\Debug\dsdf.csv") # arcpy.da.NumPyArrayToTable(wtdf,r"G:\Public\UPLAN\Calaveras\Debug\wtdf.csv") # Utilities.SavePdDataFrameToTable(df1, r"G:\Public\UPLAN\Calaveras\Debug", 'devSpaceTable.csv') # Utilities.SavePdDataFrameToTable(df2, r"G:\Public\UPLAN\Calaveras\Debug", 'gpdf.csv') # Utilities.SavePdDataFrameToTable(df3, r"G:\Public\UPLAN\Calaveras\Debug", 'dsdf.csv') # Utilities.SavePdDataFrameToTable(df4, r"G:\Public\UPLAN\Calaveras\Debug", 'wtdf.csv') # create table with developable space, gp availablity (boolean), and net weight for each parcel devSpaceTable = Utilities.MergeDataFrames([devSpaceTable, gpdf,dsdf,wtdf], str(UPConfig['BaseGeom_id'])) # devSpaceTable.set_index([UPConfig['BaseGeom_id']]) # devSpaceTable[UPConfig['BaseGeom_id']] = devSpaceTable.index #- this causes a change in parcelIDs #for debug # Utilities.SavePdDataFrameToTable(devSpaceTable, r"G:\Public\UPLAN\Calaveras\Debug", 'devSpaceTable.csv') # Utilities.SavePdDataFrameToTable(gpdf, r"G:\Public\UPLAN\Calaveras\Debug", 'gpdf.csv') # Utilities.SavePdDataFrameToTable(dsdf, r"G:\Public\UPLAN\Calaveras\Debug", 'dsdf.csv') # Utilities.SavePdDataFrameToTable(wtdf, r"G:\Public\UPLAN\Calaveras\Debug", 'wtdf.csv') # get General Plans gplans = MakeGPList(UPConfig,TimeStep) # Redev Table. if UPConfig['Redev'] != None: redevTable = os.path.join(UPConfig['paths']['dbpath'],UPConfig['paths']['dbname'],UPConfig['Redev']) flds = [UPConfig['BaseGeom_id'],UPConfig['Redev_pop'],UPConfig['Redev_emp']] redevarray = arcpy.da.TableToNumPyArray(redevTable,flds,skip_nulls=True) reDevDF = pd.DataFrame(redevarray) devSpaceTable = Utilities.MergeDataFrames([devSpaceTable, gplans,reDevDF], UPConfig['BaseGeom_id']) else: devSpaceTable = Utilities.MergeDataFrames([devSpaceTable, gplans], str(UPConfig['BaseGeom_id'])) # fix null values devSpaceTable = devSpaceTable.fillna(0) return(devSpaceTable)
def train(args, arguments): batch_time = Utilities.AverageMeter() losses = Utilities.AverageMeter() # switch to train mode arguments['detr'].train() end = time.time() train_loader_len = int(math.ceil(arguments['TADL'].shard_size / args.batch_size)) i = 0 arguments['TADL'].reset_avail_winds(arguments['epoch']) ######################################################## #_, inputs, _, targets, _ = arguments['TADL'].get_batch() #targets = transform_targets(targets) #lr_scheduler = torch.optim.lr_scheduler.StepLR(arguments['optimizer'], args.lrsp, # args.lrm) ######################################################## ######################################################## #while True: ######################################################## while i * arguments['TADL'].batch_size < arguments['TADL'].shard_size: # get the noisy inputs and the labels _, inputs, _, targets, _ = arguments['TADL'].get_batch() mean = torch.mean(inputs, 1, True) inputs = inputs-mean # zero the parameter gradients arguments['optimizer'].zero_grad() # forward + backward + optimize inputs = inputs.unsqueeze(1) outputs = arguments['detr'](inputs) ######################################################## #inputs = inputs.squeeze(1) ######################################################## # Compute the loss targets = transform_targets(targets) loss_dict = arguments['criterion'].forward(outputs=outputs, targets=targets) weight_dict = arguments['criterion'].weight_dict loss = sum(loss_dict[k] * weight_dict[k] for k in loss_dict.keys() if k in weight_dict) # compute gradient and do optimizer step loss.backward() torch.nn.utils.clip_grad_norm_(arguments['detr'].parameters(), 0.1) arguments['optimizer'].step() #if args.test: #if i > 10: #break if i%args.print_freq == 0: #if i%args.print_freq == 0 and i != 0: # Every print_freq iterations, check the loss and speed. # For best performance, it doesn't make sense to print these metrics every # iteration, since they incur an allreduce and some host<->device syncs. # Average loss across processes for logging if args.distributed: reduced_loss = Utilities.reduce_tensor(loss.data, args.world_size) else: reduced_loss = loss.data # to_python_float incurs a host<->device sync losses.update(Utilities.to_python_float(reduced_loss), args.batch_size) if not args.cpu: torch.cuda.synchronize() batch_time.update((time.time() - end)/args.print_freq, args.print_freq) end = time.time() if args.local_rank == 0: print('Epoch: [{0}][{1}/{2}]\t' 'Time {batch_time.val:.3f} ({batch_time.avg:.3f})\t' 'Speed {3:.3f} ({4:.3f})\t' 'Loss {loss.val:.10f} ({loss.avg:.4f})'.format( arguments['epoch'], i, train_loader_len, args.world_size*args.batch_size/batch_time.val, args.world_size*args.batch_size/batch_time.avg, batch_time=batch_time, loss=losses)) i += 1 ######################################################## #lr_scheduler.step() ######################################################## arguments['loss_history'].append(losses.avg) return batch_time.sum, batch_time.avg
def __init__(self, parent=None, doLoadPlugins=True, develPlugin=None): """ Constructor The Plugin Manager deals with three different plugin directories. The first is the one, that is part of eric6 (eric6/Plugins). The second one is the global plugin directory called 'eric6plugins', which is located inside the site-packages directory. The last one is the user plugin directory located inside the .eric6 directory of the users home directory. @param parent reference to the parent object (QObject) @keyparam doLoadPlugins flag indicating, that plugins should be loaded (boolean) @keyparam develPlugin filename of a plugin to be loaded for development (string) @exception PluginPathError raised to indicate an invalid plug-in path """ super(PluginManager, self).__init__(parent) self.__ui = parent self.__develPluginFile = develPlugin self.__develPluginName = None self.__inactivePluginsKey = "PluginManager/InactivePlugins" self.pluginDirs = { "eric6": os.path.join(getConfig('ericDir'), "Plugins"), "global": os.path.join(Utilities.getPythonModulesDirectory(), "eric6plugins"), "user": os.path.join(Utilities.getConfigDir(), "eric6plugins"), } self.__priorityOrder = ["eric6", "global", "user"] self.__defaultDownloadDir = os.path.join(Utilities.getConfigDir(), "Downloads") self.__activePlugins = {} self.__inactivePlugins = {} self.__onDemandActivePlugins = {} self.__onDemandInactivePlugins = {} self.__activeModules = {} self.__inactiveModules = {} self.__onDemandActiveModules = {} self.__onDemandInactiveModules = {} self.__failedModules = {} self.__foundCoreModules = [] self.__foundGlobalModules = [] self.__foundUserModules = [] self.__modulesCount = 0 pdirsExist, msg = self.__pluginDirectoriesExist() if not pdirsExist: raise PluginPathError(msg) if doLoadPlugins: if not self.__pluginModulesExist(): raise PluginModulesError self.__insertPluginsPaths() self.__loadPlugins() self.__checkPluginsDownloadDirectory() self.pluginRepositoryFile = \ os.path.join(Utilities.getConfigDir(), "PluginRepository") # attributes for the network objects self.__networkManager = QNetworkAccessManager(self) self.__networkManager.proxyAuthenticationRequired.connect( proxyAuthenticationRequired) if SSL_AVAILABLE: self.__sslErrorHandler = E5SslErrorHandler(self) self.__networkManager.sslErrors.connect(self.__sslErrors) self.__replies = []
#AvgLArQ = chainReader.ReadBranch("jet_%s_AverageLArQF"%JET_TYPE)[iJet] #LArQmean = AvgLArQ/65535. jet_pt.push_back(jetpt) jet_e.push_back(jete) jet_eta.push_back(jeteta) jet_raw_eta.push_back( chainReader.ReadBranch("jet_%s_%s_eta" % (JET_TYPE, SCALE))[iJet]) jet_phi.push_back(jetphi) jet_y.push_back(jety) jet_jvtxf.push_back(jetjvtxf) jet_origin.push_back(jetorigin) jetugly = jetugly + Utilities.TileHotSpotCleaning( chainReader.ReadBranch("RunNumber"), jeteta, jetphi, fmax, smax, verbose=False) jetugly = jetugly + Utilities.ChfCleaning( jetpt, jeteta, chf, emf, verbose=False) jet_isUgly.push_back(jetugly) # === Jet cleaning #isBadLooser,isBadLooserReason = Utilities.JetID("LooserBad", larq, negE, emf, hecf, jet_time, fmax, em_eta, chf, hecq, LArQmean) #isBadLoose,isBadLooseReason = Utilities.JetID("LooseBad", larq, negE, emf, hecf, jet_time, fmax, em_eta, chf, hecq, LArQmean) #isBadMedium,isBadMediumReason = Utilities.JetID("MediumBad", larq, negE, emf, hecf, jet_time, fmax, em_eta, chf, hecq, LArQmean) #isBadTight,isBadTightReason = Utilities.JetID("TightBad", larq, negE, emf, hecf, jet_time, fmax, em_eta, chf, hecq, LArQmean) isBadLooser = chainReader.ReadBranch("jet_%s_isBadLooseMinus" % JET_TYPE)[iJet]
def validate(args, arguments): average_precision = Utilities.AverageMeter() # switch to evaluate mode arguments['detr'].eval() end = time.time() val_loader_len = int(math.ceil(arguments['VADL'].shard_size / args.batch_size)) i = 0 arguments['VADL'].reset_avail_winds(arguments['epoch']) pred_segments = [] true_segments = [] while i * arguments['VADL'].batch_size < arguments['VADL'].shard_size: # get the noisy inputs and the labels _, inputs, _, targets, labels = arguments['TADL'].get_batch() mean = torch.mean(inputs, 1, True) inputs = inputs-mean with torch.no_grad(): # forward inputs = inputs.unsqueeze(1) outputs = arguments['detr'](inputs) for j in range(arguments['VADL'].batch_size): train_idx = int(j + i * arguments['VADL'].batch_size) probabilities = F.softmax(outputs['pred_logits'][j], dim=1) aux_pred_segments = outputs['pred_segments'][j] for probability, pred_segment in zip(probabilities.to('cpu'), aux_pred_segments.to('cpu')): #if probability[-1] < 0.9: if torch.argmax(probability) != args.num_classes: segment = [train_idx, np.argmax(probability[:-1]).item(), 1.0 - probability[-1].item(), pred_segment[0].item(), pred_segment[1].item()] pred_segments.append(segment) num_pulses = labels[j, 0] starts = targets[j, 0] widths = targets[j, 1] categories = targets[j, 3] for k in range(int(num_pulses.item())): segment = [train_idx, categories[k].item(), 1.0, starts[k].item(), widths[k].item()] true_segments.append(segment) i += 1 for threshold in np.arange(0.5, 0.95, 0.05): detection_precision=mean_average_precision(device=arguments['device'], pred_segments=pred_segments, true_segments=true_segments, iou_threshold=threshold, seg_format="mix", num_classes=1) if args.distributed: reduced_detection_precision = Utilities.reduce_tensor(detection_precision.data, args.world_size) else: reduced_detection_precision = detection_precision.data average_precision.update(Utilities.to_python_float(reduced_detection_precision)) if not args.evaluate: arguments['precision_history'].append(average_precision.avg) return average_precision.avg
def messageHandler(msgType, *args): """ Module function handling messages. @param msgType type of the message (integer, QtMsgType) @param args message handler arguments, for PyQt4 message to be shown (bytes), for PyQt5 context information (QMessageLogContext) and message to be shown (bytes) """ if len(args) == 2: context = args[0] message = args[1] else: message = args[0] if __msgHandlerDialog: try: if msgType == QtDebugMsg: messageType = QCoreApplication.translate( "E5ErrorMessage", "Debug Message:") elif msgType == QtWarningMsg: messageType = QCoreApplication.translate( "E5ErrorMessage", "Warning:") elif msgType == QtCriticalMsg: messageType = QCoreApplication.translate( "E5ErrorMessage", "Critical:") elif msgType == QtFatalMsg: messageType = QCoreApplication.translate( "E5ErrorMessage", "Fatal Error:") if isinstance(message, bytes): message = Utilities.decodeBytes(message) message = message.replace("\r\n", "<br/>")\ .replace("\n", "<br/>")\ .replace("\r", "<br/>") if len(args) == 2: msg = "<p><b>{0}</b></p><p>{1}</p><p>File: {2}</p>" \ "<p>Line: {3}</p><p>Function: {4}</p>".format( messageType, Utilities.html_uencode(message), context.file, context.line, context.function) else: msg = "<p><b>{0}</b></p><p>{1}</p>".format( messageType, Utilities.html_uencode(message)) if QThread.currentThread() == qApp.thread(): __msgHandlerDialog.showMessage(msg) else: QMetaObject.invokeMethod( __msgHandlerDialog, "showMessage", Qt.QueuedConnection, Q_ARG(str, msg)) return except RuntimeError: pass elif __origMsgHandler: __origMsgHandler(msgType, message) return if msgType == QtDebugMsg: messageType = QCoreApplication.translate( "E5ErrorMessage", "Debug Message") elif msgType == QtWarningMsg: messageType = QCoreApplication.translate( "E5ErrorMessage", "Warning") elif msgType == QtCriticalMsg: messageType = QCoreApplication.translate( "E5ErrorMessage", "Critical") elif msgType == QtFatalMsg: messageType = QCoreApplication.translate( "E5ErrorMessage", "Fatal Error") if isinstance(message, bytes): message = message.decode() if len(args) == 2: print("{0}: {1} in {2} at line {3} ({4})".format( messageType, message, context.file, context.line, context.function)) else: print("{0}: {1}".format(messageType, message))
def main(): global best_precision, args best_precision = 0 args = parse() if not len(args.data): raise Exception("error: No data set provided") args.distributed = False if 'WORLD_SIZE' in os.environ: args.distributed = int(os.environ['WORLD_SIZE']) > 1 args.gpu = 0 args.world_size = 1 if args.distributed: args.gpu = args.local_rank if not args.cpu: torch.cuda.set_device(args.gpu) torch.distributed.init_process_group(backend='gloo', init_method='env://') args.world_size = torch.distributed.get_world_size() args.total_batch_size = args.world_size * args.batch_size # Set the device device = torch.device('cpu' if args.cpu else 'cuda:' + str(args.gpu)) ####################################################################### # Start DETR contruction ####################################################################### # create DETR backbone # create backbone pulse counter if args.test: args.pulse_counter_arch = 'ResNet10' if args.local_rank==0 and args.verbose: print("=> creating backbone pulse counter '{}'".format(args.pulse_counter_arch)) if args.pulse_counter_arch == 'ResNet18': backbone_pulse_counter = rn.ResNet18_Counter() elif args.pulse_counter_arch == 'ResNet34': backbone_pulse_counter = rn.ResNet34_Counter() elif args.pulse_counter_arch == 'ResNet50': backbone_pulse_counter = rn.ResNet50_Counter() elif args.pulse_counter_arch == 'ResNet101': backbone_pulse_counter = rn.ResNet101_Counter() elif args.pulse_counter_arch == 'ResNet152': backbone_pulse_counter = rn.ResNet152_Counter() elif args.pulse_counter_arch == 'ResNet10': backbone_pulse_counter = rn.ResNet10_Counter() else: print("Unrecognized {} architecture for the backbone pulse counter" .format(args.pulse_counter_arch)) backbone_pulse_counter = backbone_pulse_counter.to(device) # create backbone feature predictor if args.test: args.feature_predictor_arch = 'ResNet10' if args.local_rank==0 and args.verbose: print("=> creating backbone feature predictor '{}'".format(args.feature_predictor_arch)) if args.feature_predictor_arch == 'ResNet18': backbone_feature_predictor = rn.ResNet18_Custom() elif args.feature_predictor_arch == 'ResNet34': backbone_feature_predictor = rn.ResNet34_Custom() elif args.feature_predictor_arch == 'ResNet50': backbone_feature_predictor = rn.ResNet50_Custom() elif args.feature_predictor_arch == 'ResNet101': backbone_feature_predictor = rn.ResNet101_Custom() elif args.feature_predictor_arch == 'ResNet152': backbone_feature_predictor = rn.ResNet152_Custom() elif args.feature_predictor_arch == 'ResNet10': backbone_feature_predictor = rn.ResNet10_Custom() else: print("Unrecognized {} architecture for the backbone feature predictor" .format(args.feature_predictor_arch)) backbone_feature_predictor = backbone_feature_predictor.to(device) # For distributed training, wrap the model with torch.nn.parallel.DistributedDataParallel. if args.distributed: if args.cpu: backbone_pulse_counter = DDP(backbone_pulse_counter) backbone_feature_predictor = DDP(backbone_feature_predictor) else: backbone_pulse_counter = DDP(backbone_pulse_counter, device_ids=[args.gpu], output_device=args.gpu) backbone_feature_predictor = DDP(backbone_feature_predictor, device_ids=[args.gpu], output_device=args.gpu) if args.verbose: print('Since we are in a distributed setting the backbone componets are replicated here in local rank {}' .format(args.local_rank)) # bring counter from a checkpoint if args.counter: # Use a local scope to avoid dangling references def bring_counter(): if os.path.isfile(args.counter): print("=> loading backbone pulse counter '{}'" .format(args.counter)) if args.cpu: checkpoint = torch.load(args.counter, map_location='cpu') else: checkpoint = torch.load(args.counter, map_location = lambda storage, loc: storage.cuda(args.gpu)) loss_history_1 = checkpoint['loss_history'] counter_error_history = checkpoint['Counter_error_history'] best_error_1 = checkpoint['best_error'] backbone_pulse_counter.load_state_dict(checkpoint['state_dict']) total_time_1 = checkpoint['total_time'] print("=> loaded counter '{}' (epoch {})" .format(args.counter, checkpoint['epoch'])) print("Counter best precision saved was {}" .format(best_error_1)) return best_error_1, backbone_pulse_counter, loss_history_1, counter_error_history, total_time_1 else: print("=> no counter found at '{}'" .format(args.counter)) best_error_1, backbone_pulse_counter, loss_history_1, counter_error_history, total_time_1 = bring_counter() else: raise Exception("error: No counter path provided") # bring predictor from a checkpoint if args.predictor: # Use a local scope to avoid dangling references def bring_predictor(): if os.path.isfile(args.predictor): print("=> loading backbone feature predictor '{}'" .format(args.predictor)) if args.cpu: checkpoint = torch.load(args.predictor, map_location='cpu') else: checkpoint = torch.load(args.predictor, map_location = lambda storage, loc: storage.cuda(args.gpu)) loss_history_2 = checkpoint['loss_history'] duration_error_history = checkpoint['duration_error_history'] amplitude_error_history = checkpoint['amplitude_error_history'] best_error_2 = checkpoint['best_error'] backbone_feature_predictor.load_state_dict(checkpoint['state_dict']) total_time_2 = checkpoint['total_time'] print("=> loaded predictor '{}' (epoch {})" .format(args.predictor, checkpoint['epoch'])) print("Predictor best precision saved was {}" .format(best_error_2)) return best_error_2, backbone_feature_predictor, loss_history_2, duration_error_history, amplitude_error_history, total_time_2 else: print("=> no predictor found at '{}'" .format(args.predictor)) best_error_2, backbone_feature_predictor, loss_history_2, duration_error_history, amplitude_error_history, total_time_2 = bring_predictor() else: raise Exception("error: No predictor path provided") # create backbone if args.local_rank==0 and args.verbose: print("=> creating backbone") if args.feature_predictor_arch == 'ResNet18': backbone=build_backbone(pulse_counter=backbone_pulse_counter, feature_predictor=backbone_feature_predictor, num_channels=512) elif args.feature_predictor_arch == 'ResNet34': backbone=build_backbone(pulse_counter=backbone_pulse_counter, feature_predictor=backbone_feature_predictor, num_channels=512) elif args.feature_predictor_arch == 'ResNet50': backbone=build_backbone(pulse_counter=backbone_pulse_counter, feature_predictor=backbone_feature_predictor, num_channels=2048) elif args.feature_predictor_arch == 'ResNet101': backbone=build_backbone(pulse_counter=backbone_pulse_counter, feature_predictor=backbone_feature_predictor, num_channels=2048) elif args.feature_predictor_arch == 'ResNet152': backbone=build_backbone(pulse_counter=backbone_pulse_counter, feature_predictor=backbone_feature_predictor, num_channels=2048) elif args.feature_predictor_arch == 'ResNet10': backbone=build_backbone(pulse_counter=backbone_pulse_counter, feature_predictor=backbone_feature_predictor, num_channels=512) else: print("Unrecognized {} architecture for the backbone feature predictor" .format(args.feature_predictor_arch)) backbone = backbone.to(device) # create DETR transformer if args.local_rank==0 and args.verbose: print("=> creating transformer") if args.test: args.transformer_hidden_dim = 64 args.transformer_num_heads = 2 args.transformer_dim_feedforward = 256 args.transformer_num_enc_layers = 2 args.transformer_num_dec_layers = 2 args.transformer_pre_norm = True transformer = build_transformer(hidden_dim=args.transformer_hidden_dim, dropout=args.transformer_dropout, nheads=args.transformer_num_heads, dim_feedforward=args.transformer_dim_feedforward, enc_layers=args.transformer_num_enc_layers, dec_layers=args.transformer_num_dec_layers, pre_norm=args.transformer_pre_norm) # create DETR in itself if args.local_rank==0 and args.verbose: print("=> creating DETR") detr = DT.DETR(backbone=backbone, transformer=transformer, num_classes=args.num_classes, num_queries=args.num_queries) detr = detr.to(device) # For distributed training, wrap the model with torch.nn.parallel.DistributedDataParallel. if args.distributed: if args.cpu: detr = DDP(detr) else: detr = DDP(detr, device_ids=[args.gpu], output_device=args.gpu) if args.verbose: print('Since we are in a distributed setting DETR model is replicated here in local rank {}' .format(args.local_rank)) # Set matcher if args.local_rank==0 and args.verbose: print("=> set Hungarian Matcher") matcher = mtchr.HungarianMatcher(cost_class=args.cost_class, cost_bsegment=args.cost_bsegment, cost_giou=args.cost_giou) # Set criterion if args.local_rank==0 and args.verbose: print("=> set criterion for the loss") weight_dict = {'loss_ce': args.loss_ce, 'loss_bsegment': args.loss_bsegment, 'loss_giou': args.loss_giou} losses = ['labels', 'segments', 'cardinality'] criterion = DT.SetCriterion(num_classes=args.num_classes, matcher=matcher, weight_dict=weight_dict, eos_coef=args.eos_coef, losses=losses) criterion = criterion.to(device) # Set optimizer optimizer = Model_Util.get_DETR_optimizer(detr, args) if args.local_rank==0 and args.verbose: print('Optimizer used for this run is {}'.format(args.optimizer)) # Set learning rate scheduler lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, args.lrsp, args.lrm) total_time = Utilities.AverageMeter() loss_history = [] precision_history = [] # Optionally resume from a checkpoint if args.resume: # Use a local scope to avoid dangling references def resume(): if os.path.isfile(args.resume): print("=> loading checkpoint '{}'" .format(args.resume)) if args.cpu: checkpoint = torch.load(args.resume, map_location='cpu') else: checkpoint = torch.load(args.resume, map_location = lambda storage, loc: storage.cuda(args.gpu)) loss_history = checkpoint['loss_history'] precision_history = checkpoint['precision_history'] start_epoch = checkpoint['epoch'] best_precision = checkpoint['best_precision'] detr.load_state_dict(checkpoint['state_dict']) criterion.load_state_dict(checkpoint['criterion']) optimizer.load_state_dict(checkpoint['optimizer']) lr_scheduler.load_state_dict(checkpoint['lr_scheduler']) total_time = checkpoint['total_time'] print("=> loaded checkpoint '{}' (epoch {})" .format(args.resume, checkpoint['epoch'])) return start_epoch, detr, criterion, optimizer, lr_scheduler, loss_history, precision_history, total_time, best_precision else: print("=> no checkpoint found at '{}'" .format(args.resume)) args.start_epoch, detr, criterion, optimizer, lr_scheduler, loss_history, precision_history, total_time, best_precision = resume() # Data loading code if len(args.data) == 1: traindir = os.path.join(args.data[0], 'train') valdir = os.path.join(args.data[0], 'val') else: traindir = args.data[0] valdir= args.data[1] if args.test: training_f = h5py.File(traindir + '/train_toy.h5', 'r') validation_f = h5py.File(valdir + '/validation_toy.h5', 'r') else: training_f = h5py.File(traindir + '/train.h5', 'r') validation_f = h5py.File(valdir + '/validation.h5', 'r') # this is the dataset for training sampling_rate = 10000 # This is the number of samples per second of the signals in the dataset if args.test: number_of_concentrations = 2 # This is the number of different concentrations in the dataset number_of_durations = 2 # This is the number of different translocation durations per concentration in the dataset number_of_diameters = 4 # This is the number of different translocation durations per concentration in the dataset window = 0.5 # This is the time window in seconds length = 20 # This is the time of a complete signal for certain concentration and duration else: number_of_concentrations = 20 # This is the number of different concentrations in the dataset number_of_durations = 5 # This is the number of different translocation durations per concentration in the dataset number_of_diameters = 15 # This is the number of different translocation durations per concentration in the dataset window = 0.5 # This is the time window in seconds length = 20 # This is the time of a complete signal for certain concentration and duration # Training Artificial Data Loader TADL = Artificial_DataLoader(args.world_size, args.local_rank, device, training_f, sampling_rate, number_of_concentrations, number_of_durations, number_of_diameters, window, length, args.batch_size) # this is the dataset for validating if args.test: number_of_concentrations = 2 # This is the number of different concentrations in the dataset number_of_durations = 2 # This is the number of different translocation durations per concentration in the dataset number_of_diameters = 4 # This is the number of different translocation durations per concentration in the dataset window = 0.5 # This is the time window in seconds length = 10 # This is the time of a complete signal for certain concentration and duration else: number_of_concentrations = 20 # This is the number of different concentrations in the dataset number_of_durations = 5 # This is the number of different translocation durations per concentration in the dataset number_of_diameters = 15 # This is the number of different translocation durations per concentration in the dataset window = 0.5 # This is the time window in seconds length = 10 # This is the time of a complete signal for certain concentration and duration # Validating Artificial Data Loader VADL = Artificial_DataLoader(args.world_size, args.local_rank, device, validation_f, sampling_rate, number_of_concentrations, number_of_durations, number_of_diameters, window, length, args.batch_size) if args.verbose: print('From rank {} training shard size is {}'. format(args.local_rank, TADL.get_number_of_avail_windows())) print('From rank {} validation shard size is {}'. format(args.local_rank, VADL.get_number_of_avail_windows())) if args.run: arguments = {'model': detr, 'device': device, 'epoch': 0, 'VADL': VADL} if args.local_rank == 0: run_model(args, arguments) return #if args.statistics: #arguments = {'model': model, #'device': device, #'epoch': 0, #'VADL': VADL} #[duration_errors, amplitude_errors] = compute_error_stats(args, arguments) #if args.local_rank == 0: #plot_stats(VADL, duration_errors, amplitude_errors) #return #if args.evaluate: #arguments = {'model': model, #'device': device, #'epoch': 0, #'VADL': VADL} #[duration_error, amplitude_error] = validate(args, arguments) #print('##Duration error {0}\n' #'##Amplitude error {1}'.format( #duration_error, #amplitude_error)) #return if args.plot_training_history and args.local_rank == 0: Model_Util.plot_detector_stats(loss_history, precision_history) hours = int(total_time.sum / 3600) minutes = int((total_time.sum % 3600) / 60) seconds = int((total_time.sum % 3600) % 60) print('The total training time was {} hours {} minutes and {} seconds' .format(hours, minutes, seconds)) hours = int(total_time.avg / 3600) minutes = int((total_time.avg % 3600) / 60) seconds = int((total_time.avg % 3600) % 60) print('while the average time during one epoch of training was {} hours {} minutes and {} seconds' .format(hours, minutes, seconds)) return for epoch in range(args.start_epoch, args.epochs): arguments = {'detr': detr, 'criterion': criterion, 'optimizer': optimizer, 'device': device, 'epoch': epoch, 'TADL': TADL, 'VADL': VADL, 'loss_history': loss_history, 'precision_history': precision_history} # train for one epoch epoch_time, avg_batch_time = train(args, arguments) total_time.update(epoch_time) # validate every val_freq epochs if epoch%args.val_freq == 0 and epoch != 0: # evaluate on validation set print("\nValidating ...\nComputing mean average precision (mAP) for epoch {}" .format(epoch)) precision = validate(args, arguments) else: precision = best_precision #if args.test: #break lr_scheduler.step() # remember the best detr and save checkpoint if args.local_rank == 0: if epoch%args.val_freq == 0: print('From validation we have precision is {} while best_precision is {}'.format(precision, best_precision)) is_best = precision > best_precision best_precision = max(precision, best_precision) Model_Util.save_checkpoint({ 'arch': 'DETR_' + args.feature_predictor_arch, 'epoch': epoch + 1, 'best_precision': best_precision, 'state_dict': detr.state_dict(), 'criterion': criterion.state_dict(), 'optimizer': optimizer.state_dict(), 'loss_history': loss_history, 'precision_history': precision_history, 'lr_scheduler': lr_scheduler.state_dict(), 'total_time': total_time }, is_best) print('##Detector precision {0}\n' '##Perf {1}'.format( precision, args.total_batch_size / avg_batch_time))