fileUtil.copyFile(filePath.replace(filePath, build_config.inputfolder + "/" + filePath), build_config.minFolder) fileUtil.copyFile(filePath.replace(filePath, build_config.inputfolder + "/" + filePath), build_config.standaloneFolder) fileUtil.copyFile(filePath.replace(filePath, build_config.inputfolder + "/" + filePath), build_config.standaloneFolderMin) for noMin in cssFilesNotToMinify: filePath = re.findall("href=[\"'](.+?)[\"']", noMin) filePath = filePath[0] fileUtil.copyFile(filePath.replace(filePath, build_config.inputfolder + "/" + filePath), build_config.unminFolder) fileUtil.copyFile(filePath.replace(filePath, build_config.inputfolder + "/" + filePath), build_config.minFolder) fileUtil.copyFile(filePath.replace(filePath, build_config.inputfolder + "/" + filePath), build_config.standaloneFolder) fileUtil.copyFile(filePath.replace(filePath, build_config.inputfolder + "/" + filePath), build_config.standaloneFolderMin) # 8 - Identify js/css files that we should group separately to our main concatenated files, e.g. for IE specific styles/scripts, or scripts that rely on particular browser functionality jsGroups = re.findall('<script.+?data-type=[\'"](.+?)[\'"].+?script>', releaseHtml) utils.removeDuplicates(jsGroups) jsGroups = utils.getRealGroups(jsGroups) cssGroups = re.findall('<link.+?data-type=[\'"](.+?)[\'"].+?/>', releaseHtml) utils.removeDuplicates(cssGroups) cssGroups = utils.getRealGroups(cssGroups) # 9 - if there are any files in the config file that have been hardcoded to be copied across (e.g. if they are only injected via js rather than <script> tag), then copy these across to all target folders (these won't be minimised) # to-do - minimise these files if they're js/css files... for cFile in build_config.copyFiles: fileUtil.copyFile(cFile.replace(cFile, build_config.inputfolder + "/" + cFile), build_config.unminFolder) fileUtil.copyFile(cFile.replace(cFile, build_config.inputfolder + "/" + cFile), build_config.minFolder) fileUtil.copyFile(cFile.replace(cFile, build_config.inputfolder + "/" + cFile), build_config.standaloneFolder) fileUtil.copyFile(cFile.replace(cFile, build_config.inputfolder + "/" + cFile), build_config.standaloneFolderMin)
def build_data( self, dataset_name=None, shared_vocab_fields=None, max_len=None, ): _fields = { field.name: ( field.field if field.field != None else field.name, Field( sequential=field.sequential, tokenize=word_tokenize if field.tokenize is None else field.tokenize, lower=field.lowercase, fix_length=field.fix_length, max_len=max_len, init_token=bos_token if (field.sos and field.sequential) else None, eos_token=eos_token if (field.eos and field.sequential) else None, include_lengths=field.include_lengths, batch_first=True, tokenizer=field.tokenizer, use_vocab=field.use_vocab, pad_token=field.pad_token, postprocessing=field.postprocess, ), ) for field in self.dataset_fields } if shared_vocab_fields: if isinstance(shared_vocab_fields[0], list) == False: shared_vocab_fields = [shared_vocab_fields] self.share_vocab_fields(_fields, shared_vocab_fields) if shared_vocab_fields: assert (_fields[shared_vocab_fields[0][0]][1] == _fields[ shared_vocab_fields[0][1]][1]) seperate_fields = removeDuplicates([_fields[i][1] for i in _fields]) print("Loading", dataset_name) self.train, self.valid, self.test = data.TabularDataset.splits( path=os.path.join(self.dataset_path, dataset_name), train="_train.json", validation="_valid.json", test="_test.json", format="json", fields=_fields, ) print("Dataset:", dataset_name, "loaded") for f in seperate_fields: try: if f.use_vocab: f.build_vocab( self.train, self.valid, self.test, specials=special_tokens_list, min_freq=1, max_size=10000 if f.tokenize == word_tokenize else 100000, ) except: pass self.vocab = {} for key in _fields: try: self.vocab[_fields[key][0]] = _fields[key][1].vocab except: self.vocab[_fields[key][0]] = _fields[key][1].tokenizer self.vocab = DotMap(self.vocab)
class Meta: model = TransportHTTPS fields = removeDuplicates(TransportHTTPSerializer.Meta.fields, TransportAuthSerializer.Meta.fields)
def createConcatenatedFileLinksInHtml(filePath, htmlRef, removeFilesFromBuild, removeStaticPath, isMin): html = htmlRef #remove all local only (if removeFilesFromBuild) # to-do: combine into single function allJsScripts = re.findall( '<script.+?[src=].+?[type=].+?[text/javascript].+?<\/script>', html) jsTypesToCombine = [] for jsScript in allJsScripts: dataType = re.findall('data-type=[\'"](.+?)[\'"]', jsScript) if len(dataType) > 0: if dataType[0] == build_config.removeFromBuild: if removeFilesFromBuild: html = re.sub(jsScript, "", html) else: replaceStr = re.sub('data-type=[\'"].+?[\'"]', '', jsScript) html = re.sub(jsScript, replaceStr, html) elif dataType[0] == build_config.doNotMinify: replaceStr = re.sub('data-type=[\'"].+?[\'"]', '', jsScript) html = re.sub(jsScript, replaceStr, html) else: #it's a group (we've already put out main js into a group) jsTypesToCombine.append(dataType[0]) utils.removeDuplicates(jsTypesToCombine) for jsDataType in jsTypesToCombine: allRelatedScripts = re.findall( '<script.+?data-type=[\'"]+?' + re.escape(jsDataType) + '.+?[\'"].+?<\/script>', html) if isMin: filePath = build_config.jsOutputFolder + "/" + jsDataType + ".min.js" html = re.sub( allRelatedScripts[0], '<script charset="utf-8" type="text/javascript" src="' + filePath + '"></script>\n', html) else: filePath = build_config.jsOutputFolder + "/" + jsDataType + ".js" html = re.sub( allRelatedScripts[0], '<script charset="utf-8" data-type="to-minify" type="text/javascript" src="' + filePath + '"></script>\n', html) for script in allRelatedScripts: html = re.sub(script, "", html) allCssScripts = re.findall( '<link.+?[href=].+?[rel=\"stylesheet\"].+?[type=\"text\/css\"].+?\/>', html) cssTypesToCombine = [] for cssScript in allCssScripts: dataType = re.findall('data-type=[\'"](.+?)[\'"]', cssScript) if len(dataType) > 0: if dataType[0] == build_config.removeFromBuild: if removeFilesFromBuild: html = re.sub(cssScript, "", html) else: replaceStr = re.sub('data-type=[\'"].+?[\'"]', '', cssScript) html = re.sub(cssScript, replaceStr, html) elif dataType[0] == build_config.doNotMinify: replaceStr = re.sub('data-type=[\'"].+?[\'"]', '', cssScript) html = re.sub(cssScript, replaceStr, html) else: #it's a group (we've already put out main js into a group) cssTypesToCombine.append(dataType[0]) utils.removeDuplicates(cssTypesToCombine) for cssDataType in cssTypesToCombine: allRelatedScripts = re.findall( '<link.+?data-type=[\'"]+?' + re.escape(cssDataType) + '.+?[\'"].+?\/>', html) if isMin: filePath = build_config.cssOutputFolder + "/" + cssDataType + ".min.css" html = re.sub( allRelatedScripts[0], '<link charset="utf-8" rel="stylesheet" href="' + filePath + '"/>\n', html) else: filePath = build_config.cssOutputFolder + "/" + cssDataType + ".css" html = re.sub( allRelatedScripts[0], '<link charset="utf-8" data-type="to-minify" rel="stylesheet" href="' + filePath + '"/>\n', html) for script in allRelatedScripts: html = re.sub(script, "", html) if removeStaticPath: html = re.sub(build_config.staticPath, "", html) return html
def createConcatenatedFileLinksInHtml(filePath, htmlRef, removeFilesFromBuild, removeStaticPath, isMin): html = htmlRef #remove all local only (if removeFilesFromBuild) # to-do: combine into single function allJsScripts = re.findall('<script.+?[src=].+?[type=].+?[text/javascript].+?<\/script>', html) jsTypesToCombine = [] for jsScript in allJsScripts: dataType = re.findall('data-type=[\'"](.+?)[\'"]', jsScript) if len(dataType) > 0: if dataType[0] == build_config.removeFromBuild: if removeFilesFromBuild: html = re.sub(jsScript, "", html) else: replaceStr = re.sub('data-type=[\'"].+?[\'"]', '', jsScript) html = re.sub(jsScript, replaceStr, html) elif dataType[0] == build_config.doNotMinify: replaceStr = re.sub('data-type=[\'"].+?[\'"]', '', jsScript) html = re.sub(jsScript, replaceStr, html) else: #it's a group (we've already put out main js into a group) jsTypesToCombine.append(dataType[0]) utils.removeDuplicates(jsTypesToCombine) for jsDataType in jsTypesToCombine: allRelatedScripts = re.findall('<script.+?data-type=[\'"]+?' + re.escape(jsDataType) + '.+?[\'"].+?<\/script>', html) if isMin: filePath = build_config.jsOutputFolder + "/" + jsDataType + ".min.js" html = re.sub(allRelatedScripts[0], '<script charset="utf-8" type="text/javascript" src="' + filePath + '"></script>\n', html) else: filePath = build_config.jsOutputFolder + "/" + jsDataType + ".js" html = re.sub(allRelatedScripts[0], '<script charset="utf-8" data-type="to-minify" type="text/javascript" src="' + filePath + '"></script>\n', html) for script in allRelatedScripts: html = re.sub(script, "", html) allCssScripts = re.findall('<link.+?[href=].+?[rel=\"stylesheet\"].+?[type=\"text\/css\"].+?\/>', html) cssTypesToCombine = [] for cssScript in allCssScripts: dataType = re.findall('data-type=[\'"](.+?)[\'"]', cssScript) if len(dataType) > 0: if dataType[0] == build_config.removeFromBuild: if removeFilesFromBuild: html = re.sub(cssScript, "", html) else: replaceStr = re.sub('data-type=[\'"].+?[\'"]', '', cssScript) html = re.sub(cssScript, replaceStr, html) elif dataType[0] == build_config.doNotMinify: replaceStr = re.sub('data-type=[\'"].+?[\'"]', '', cssScript) html = re.sub(cssScript, replaceStr, html) else: #it's a group (we've already put out main js into a group) cssTypesToCombine.append(dataType[0]) utils.removeDuplicates(cssTypesToCombine) for cssDataType in cssTypesToCombine: allRelatedScripts = re.findall('<link.+?data-type=[\'"]+?' + re.escape(cssDataType) + '.+?[\'"].+?\/>', html) if isMin: filePath = build_config.cssOutputFolder + "/" + cssDataType + ".min.css" html = re.sub(allRelatedScripts[0], '<link charset="utf-8" rel="stylesheet" href="' + filePath + '"/>\n', html) else: filePath = build_config.cssOutputFolder + "/" + cssDataType + ".css" html = re.sub(allRelatedScripts[0], '<link charset="utf-8" data-type="to-minify" rel="stylesheet" href="' + filePath + '"/>\n', html) for script in allRelatedScripts: html = re.sub(script, "", html) if removeStaticPath: html = re.sub(build_config.staticPath, "", html) return html
archiveName="archive.json" archive_dict = loadEvents(PATH_TO_DATA,archiveName) os.chdir(PATH_TO_CODE) incidentList=[] incident_nodes=[] for e in archive_dict: #each e is another event incident_nodes.append(FireNode(e["latitude"],e["longitude"],"event",0,e["acq_date"],e["confidence"],e["frp"])) incident_nodes[-1].calculateWeight(MAX_SEV,MAX_FRP,MAX_POP_DENSITY) incidentList.append((e["latitude"],e["longitude"])) SIZE_OF_CLUSTER=5 mean_cluster_nodes=[] cluster_means=[] incident_nodes=removeDuplicates(incident_nodes) copy_incident_nodes=incident_nodes[:] # for testing next_one=incident_nodes[-1] sizes=[] mean_nodes=[] while len(incident_nodes)>=SIZE_OF_CLUSTER: genesis_cluster=Cluster() next_one,incident_nodes=genesis_cluster.form_cluster(incident_nodes[-1],SIZE_OF_CLUSTER,incident_nodes) cluster_nodes=[] for node in genesis_cluster.nodes_in_cluster: cluster_nodes.append((node.lat,node.long)) try: incident_nodes.remove(node) #print("Removing") except Exception as e: print(str(e))