def copy_static_files(): static_func = info("Copy static files") folders = [ x for x in glob('static/*') if os.path.join("static", "css") not in x ] os.mkdir('build/static') for f in folders: cp.cp(f, 'build/' + f) static_func("Done", True)
def copy_template_files(): template_func = info("Copying Template Files") template_files = itertools.chain.from_iterable( [glob(g) for g in template_file_globs]) for f in template_files: dir_name = os.path.dirname(f) if dir_name != '': os.makedirs(os.path.join('build/', dir_name), exist_ok=True) cp.cp(f, 'build/' + f) template_func("Done", True)
def copy_python_files(): python_func = info("Copying Python Files") python_files = itertools.chain.from_iterable( [glob(g) for g in python_file_globs]) for f in python_files: if f == 'build.py': continue dir_name = os.path.dirname(f) if dir_name != '': os.makedirs(os.path.join('build/', dir_name), exist_ok=True) cp.cp(f, 'build/' + f) python_func("Done", True)
def mv(src, dst): import cp import rm cp.cp(src, dst) rm.rm(src)
batchsize = 40 with open('ktensor_noise_1em3.pickle', 'rb') as f: p = pickle.load(f) size = p['size'] rank = p['rank'] train_input = torch.LongTensor(p['train']['indexes']).t() train_value = torch.Tensor(p['train']['values']) #train_norm = torch.norm(train_value).item() test_input = torch.LongTensor(p['test']['indexes']).t() test_value = torch.Tensor(p['test']['values']) #test_norm = torch.norm(test_value).item() model = cp(size, rank) train_loader = DataLoader(TensorDataset(train_input, train_value), batch_size=batchsize, shuffle=True) test_loader = DataLoader(TensorDataset(test_input, test_value), batch_size=batchsize, shuffle=True) use_cuda = False kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} # class MNIST_index(datasets.MNIST): # def __getitem__(self, index): # img, target = super(MNIST_index, self).__getitem__(index) # return (img, target, index) criterion = nn.MSELoss()
def get_cp(self, tabfile): "Function returns the cp object generated from the passed in tab file." return cp.cp(tabfile)
if_e() elif send == 'z': if_z() elif send == '1': if_1() elif send == '2': if_2() elif send == '3': if_3() elif send == '4': if_4() elif send == '5': if_5() else: if_else() if os_type().upper() == 'WINDOWS': screensize() features() download() cp() main() elif os_type().upper() == 'LINUX': print(c.END + c.BLUE + 'support only' + c.GREEN + ' windows' + c.RED + ' , ' + c.GREEN + 'GNU/Linux' + c.BLUE + ' version coming soon ' + c.RED + '.' + c.GREEN + '.' + c.BLUE + '.') xinput('') else: pass
if os.name != 'nt': print("You are not using Windows.Termulator is for Windows systems only") exit(1) user = os.getlogin() while 1: try: q = input(user + '$') q = q.split() print(q) except KeyboardInterrupt: exit(1) try: if q[0] == 'pwd': pwd.pwd() elif q[0] == "ls": ls.ls() elif q[0] == "clear": clear.clear() elif q[0] == "cd": cd.cd(q) elif q[0] == "cp": cp.cp(q) elif q[0] == "mv": mv.mv(q) elif q[0] == "touch": touch.touch(q) else: print(q[0] + " is not a recognizable command") # default except IndexError: continue
print('this is the train loop:') print(i) ## the data format that sess.run get is tensor,not ndarray Xtemp1a, Ytemp1a = sess.run([featuretrain, labeltrain]) ### here we have to concate all the Xtemp1&Ttemp1 into Xtrain&Ytrain ## how to concate c1,c2,...,c5 into a densefeature (named featuretrain/test)?? ## coding here # convert np.ndarray into string so that can be fed into SVM # does Y need reshape? #print('Xtemp1a and its shape') #print(Xtemp1a.shape) ## here d=64,s=14 ## note that for AlexNet,the conv5 layer got a tensor of 14*14*256 Xtemp1 = cp(Xtemp1a, 64, 14, batch_size) Ytemp1 = np.array(Ytemp1a) #print('after cp, Xtemp1 and its shape:') #print(Xtemp1.shape) ### if else if i == 0: #print('this is for the loop i=0') Xtrain = Xtemp1 Ytrain = Ytemp1 else: ## horizontal: np.hstack; vertical:np.vstack #print('in the loop, the shape of Xtemp:') #print(Xtemp1.shape)
def compress_css(npm_module_root: str): sumfile_func = info("Creating sum.css") css = glob('static/css/*') sumfile = "" for c in css: with open(c, 'r') as f: sumfile += f.read() os.mkdir('build-staging/') with open('build-staging/sum.css', 'w') as f: f.write(sumfile) cp.cp('build-staging/sum.css', 'build-staging/sum-dce.css') sumfile_func("Done", True) '''purgecss --css build/static/staging/sum-lean.css --content templates/**/* templates/*.html -o build/static/staging/''' html = [ val for val in glob("templates/**/*", recursive=True) if not os.path.isdir(val) ] purge_func = info("CSS Dead Code Elimination") purge_args = [ 'purgecss', '--css', 'build-staging//sum-dce.css', '--content', *html, '-o', 'build-staging//' ] purge_result = subprocess.run(purge_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, shell=windows) if purge_result.returncode != 0: command = " ".join(purge_args) purge_func( f"error while running \'{command}\':\n{purge_result.stdout.decode('utf8')}", False) purge_func("Done", True) '''NODE_PATH=/usr/lib/node_modules postcss build-staging/sum-dce.css --config postcss.config.js --no-map -o build-staging/sum-min.css''' minify_func = info("CSS Minifcation") minify_args = [ 'postcss', 'build-staging/sum-dce.css', '--config', 'postcss.config.js', '--no-map', '-o', 'build-staging/sum-min.css' ] current_environ = os.environ.copy() if windows: current_environ.update( {'NODE_PATH': os.path.join(npm_module_root, 'node_modules')}) else: current_environ.update( {'NODE_PATH': os.path.join(npm_module_root, 'lib/node_modules')}) minify_result = subprocess.run(minify_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=current_environ, shell=windows) if minify_result.returncode != 0: command = " ".join(minify_args) minify_func( f"error while running \'{command}\':\n{minify_result.stderr.decode('utf8')}", False) minify_func("Done", True) # # copy css # css_copy_func = info("Copy CSS") os.makedirs('build/static/css', exist_ok=True) cp.cp('build-staging/sum-min.css', 'build/static/css/sum.css') css_copy_func("Done", True)
def get_cp(self, tabfile): return cp.cp(tabfile)
def all_chem_adjust(self, mz_heat, cp_heat, output): """Does vector compression, ignoring biological activity. Uses all baskets from a run, not just those that are within a biological cutoff range. """ new_cp = cp.cp(None) for feat in cp_heat.features(): new_cp[str(feat)] = cp.feature(str(feat)) runcount = 0 n = 0 av_dist = 0 widgets = ['VectorMove: ', Percentage(), ' ', Bar(marker=RotatingMarker()), ' ',\ ETA(), ' ', FileTransferSpeed()] pbar = ProgressBar(widgets=widgets, maxval=len(cp_heat.fingerprints())).start() largest_scaler = None #This is a dictionary of run (as string) - [run_vector, add_vector] pairs add_vectors = dict() for run in cp_heat.fingerprints(): output.write("\t" + str(run) + "\n") runcount += 1 pbar.update(runcount) labels = run.keys() #This is the original vector fingerprint in log scale run_vec = numpy.log(run.values(), dtype=float) print run_vec #This is going to be a list of vectors, one from each basket, in log scale bask_vectors = [] #This is the number of vectors, one for each connection, with multiple connections per basket vector_num = 0 for bask in mz_heat.grab_basks(str(run)): #If there's only one run, then the vector has nothing to connect to if len(bask.keys()) <= 1: continue #This is the average value of the scaler, for use in line plots later av_scaler = 0.0 for connect_run in bask.keys(): cprun = connect_run.replace("_", "") #Don't connect the query run to itself, that's not useful if cprun == str(run): continue #If the run isn't in the cp_heatmap, just continue; that means it also wasn't used for creating synthetic fingerprints if cprun not in cp_heat.map: continue #Get the vector difference between the target and the source, but it's in log scale. Also, make sure label values are # in the same order between the vectors. Remember log scale! vec_dif = numpy.log(numpy.array([cp_heat[cprun][val] for val in labels]) - run_vec) scaler = self.bask_prob(bask, cp_heat[cprun]) + self.bask_prob(bask, run) print scaler if scaler >= largest_scaler: largest_scaler = scaler av_scaler = numpy.logaddexp(av_scaler, scaler) bask_vectors.append(vec_dif + scaler) print "bask_ind", vec_dif+scaler vector_num += 1 if not vector_num == 0: output.write("\t\t\t{}; {}\n".format(str(bask), numpy.exp(av_scaler) / vector_num)) #This is the sum of the basket vectors, still in log scale print "all", bask_vectors add_vector = logsumexp(bask_vectors, axis=0) print "summed", add_vector if not vector_num == 0: add_vector -= numpy.log(vector_num) print "averaged", add_vector print 'large', largest_scaler return # if not largest_scaler == 0: # add_vector /= 2 * largest_scaler add_vectors[str(run)] = [run_vec, add_vector] for run, (run_vec, add_vector) in add_vectors.items(): add_vector = numpy.exp(add_vector - (numpy.log(2) + largest_scaler)) run_vec = numpy.exp(run_vec) + add_vector new_cp[str(run)] = cp.fingerprint(str(run)) for param, value in zip(labels, run_vec): new_cp[param][str(run)] = new_cp[str(run)][param] = value output.write("\t\tRun Movement: {}\n".format(numpy.sqrt(add_vector.dot(add_vector)))) av_dist += numpy.sqrt(add_vector.dot(add_vector)) n += 1 pbar.finish() output.write("Average Movement: " + str(av_dist / n) + "\n") return new_cp
def chem_adjust(self, mz_heat, cp_heat, output): new_cp = cp.cp(None) for feat in cp_heat.features(): new_cp[str(feat)] = cp.feature(str(feat)) runcount = 0 n = 0 av_dist = 0 widgets = ['VectorMove: ', Percentage(), ' ', Bar(marker=RotatingMarker()), ' ',\ ETA(), ' ', FileTransferSpeed()] pbar = ProgressBar(widgets=widgets, maxval=len(cp_heat.fingerprints())).start() for run in cp_heat.fingerprints(): output.write("\t" + str(run) + "\n") runcount += 1 pbar.update(runcount) all_basks = [bask for bask in mz_heat.grab_basks(str(run))] inruns = set([inrun + "_" for inrun in cp_heat.cluster(str(run), max_tolerance=0.5, min_tolerance=0.65)]) # antiruns = set([antirun + "_" for antirun in cp_heat.anticluster(str(run), pmax=-0.2, fraction=1)]) basks = [] for bask in all_basks: bruns = set(bask.keys()) # if len(bruns & antiruns) > 0: # continue if len(bruns & inruns) < 2: continue basks.append(bask) labels = run.keys() run_vec = numpy.array(run.values(), dtype=float) add_vector = numpy.zeros(len(labels)) vector_num = 0 largest_scaler = 0.0 for bask in basks: av_scaler = 0.0 for connect_run in bask.keys(): cprun = connect_run.replace("_", "") if cprun == str(run): continue if cprun not in cp_heat.map: continue vec_dif = numpy.array([cp_heat[cprun][val] for val in labels]) - run_vec scaler = self.bask_prob(bask, cp_heat[cprun]) * self.bask_prob(bask, run) if scaler >= largest_scaler: largest_scaler = scaler av_scaler += scaler add_vector += vec_dif * scaler vector_num += 1 if not vector_num == 0: av_scaler /= vector_num output.write("\t\t\t{}; {}\n".format(str(bask), av_scaler)) if not vector_num == 0: add_vector /= vector_num if not largest_scaler == 0: add_vector /= 2 * largest_scaler run_vec += add_vector new_cp[str(run)] = cp.fingerprint(str(run)) for param, value in zip(labels, run_vec): new_cp[param][str(run)] = new_cp[str(run)][param] = value output.write("\t\tRun Movement: {}\n".format(numpy.sqrt(add_vector.dot(add_vector)))) av_dist += numpy.sqrt(add_vector.dot(add_vector)) n += 1 pbar.finish() output.write("Average Movement: " + str(av_dist / n) + "\n") return new_cp