fileUtil.copyFile(filePath.replace(filePath, build_config.inputfolder + "/" + filePath), build_config.minFolder)
fileUtil.copyFile(filePath.replace(filePath, build_config.inputfolder + "/" + filePath), build_config.standaloneFolder)
fileUtil.copyFile(filePath.replace(filePath, build_config.inputfolder + "/" + filePath), build_config.standaloneFolderMin)
for noMin in cssFilesNotToMinify:
filePath = re.findall("href=[\"'](.+?)[\"']", noMin)
filePath = filePath[0]
fileUtil.copyFile(filePath.replace(filePath, build_config.inputfolder + "/" + filePath), build_config.unminFolder)
fileUtil.copyFile(filePath.replace(filePath, build_config.inputfolder + "/" + filePath), build_config.minFolder)
fileUtil.copyFile(filePath.replace(filePath, build_config.inputfolder + "/" + filePath), build_config.standaloneFolder)
fileUtil.copyFile(filePath.replace(filePath, build_config.inputfolder + "/" + filePath), build_config.standaloneFolderMin)
# 8 - Identify js/css files that we should group separately to our main concatenated files, e.g. for IE specific styles/scripts, or scripts that rely on particular browser functionality
jsGroups = re.findall('<script.+?data-type=[\'"](.+?)[\'"].+?script>', releaseHtml)
utils.removeDuplicates(jsGroups)
jsGroups = utils.getRealGroups(jsGroups)
cssGroups = re.findall('<link.+?data-type=[\'"](.+?)[\'"].+?/>', releaseHtml)
utils.removeDuplicates(cssGroups)
cssGroups = utils.getRealGroups(cssGroups)
# 9 - if there are any files in the config file that have been hardcoded to be copied across (e.g. if they are only injected via js rather than <script> tag), then copy these across to all target folders (these won't be minimised)
# to-do - minimise these files if they're js/css files...
for cFile in build_config.copyFiles:
fileUtil.copyFile(cFile.replace(cFile, build_config.inputfolder + "/" + cFile), build_config.unminFolder)
fileUtil.copyFile(cFile.replace(cFile, build_config.inputfolder + "/" + cFile), build_config.minFolder)
fileUtil.copyFile(cFile.replace(cFile, build_config.inputfolder + "/" + cFile), build_config.standaloneFolder)
fileUtil.copyFile(cFile.replace(cFile, build_config.inputfolder + "/" + cFile), build_config.standaloneFolderMin)
def build_data(
self,
dataset_name=None,
shared_vocab_fields=None,
max_len=None,
):
_fields = {
field.name: (
field.field if field.field != None else field.name,
Field(
sequential=field.sequential,
tokenize=word_tokenize
if field.tokenize is None else field.tokenize,
lower=field.lowercase,
fix_length=field.fix_length,
max_len=max_len,
init_token=bos_token if
(field.sos and field.sequential) else None,
eos_token=eos_token if
(field.eos and field.sequential) else None,
include_lengths=field.include_lengths,
batch_first=True,
tokenizer=field.tokenizer,
use_vocab=field.use_vocab,
pad_token=field.pad_token,
postprocessing=field.postprocess,
),
)
for field in self.dataset_fields
}
if shared_vocab_fields:
if isinstance(shared_vocab_fields[0], list) == False:
shared_vocab_fields = [shared_vocab_fields]
self.share_vocab_fields(_fields, shared_vocab_fields)
if shared_vocab_fields:
assert (_fields[shared_vocab_fields[0][0]][1] == _fields[
shared_vocab_fields[0][1]][1])
seperate_fields = removeDuplicates([_fields[i][1] for i in _fields])
print("Loading", dataset_name)
self.train, self.valid, self.test = data.TabularDataset.splits(
path=os.path.join(self.dataset_path, dataset_name),
train="_train.json",
validation="_valid.json",
test="_test.json",
format="json",
fields=_fields,
)
print("Dataset:", dataset_name, "loaded")
for f in seperate_fields:
try:
if f.use_vocab:
f.build_vocab(
self.train,
self.valid,
self.test,
specials=special_tokens_list,
min_freq=1,
max_size=10000
if f.tokenize == word_tokenize else 100000,
)
except:
pass
self.vocab = {}
for key in _fields:
try:
self.vocab[_fields[key][0]] = _fields[key][1].vocab
except:
self.vocab[_fields[key][0]] = _fields[key][1].tokenizer
self.vocab = DotMap(self.vocab)
class Meta:
model = TransportHTTPS
fields = removeDuplicates(TransportHTTPSerializer.Meta.fields,
TransportAuthSerializer.Meta.fields)
def createConcatenatedFileLinksInHtml(filePath, htmlRef, removeFilesFromBuild,
removeStaticPath, isMin):
html = htmlRef
#remove all local only (if removeFilesFromBuild)
# to-do: combine into single function
allJsScripts = re.findall(
'<script.+?[src=].+?[type=].+?[text/javascript].+?<\/script>', html)
jsTypesToCombine = []
for jsScript in allJsScripts:
dataType = re.findall('data-type=[\'"](.+?)[\'"]', jsScript)
if len(dataType) > 0:
if dataType[0] == build_config.removeFromBuild:
if removeFilesFromBuild:
html = re.sub(jsScript, "", html)
else:
replaceStr = re.sub('data-type=[\'"].+?[\'"]', '',
jsScript)
html = re.sub(jsScript, replaceStr, html)
elif dataType[0] == build_config.doNotMinify:
replaceStr = re.sub('data-type=[\'"].+?[\'"]', '', jsScript)
html = re.sub(jsScript, replaceStr, html)
else: #it's a group (we've already put out main js into a group)
jsTypesToCombine.append(dataType[0])
utils.removeDuplicates(jsTypesToCombine)
for jsDataType in jsTypesToCombine:
allRelatedScripts = re.findall(
'<script.+?data-type=[\'"]+?' + re.escape(jsDataType) +
'.+?[\'"].+?<\/script>', html)
if isMin:
filePath = build_config.jsOutputFolder + "/" + jsDataType + ".min.js"
html = re.sub(
allRelatedScripts[0],
'<script charset="utf-8" type="text/javascript" src="' +
filePath + '"></script>\n', html)
else:
filePath = build_config.jsOutputFolder + "/" + jsDataType + ".js"
html = re.sub(
allRelatedScripts[0],
'<script charset="utf-8" data-type="to-minify" type="text/javascript" src="'
+ filePath + '"></script>\n', html)
for script in allRelatedScripts:
html = re.sub(script, "", html)
allCssScripts = re.findall(
'<link.+?[href=].+?[rel=\"stylesheet\"].+?[type=\"text\/css\"].+?\/>',
html)
cssTypesToCombine = []
for cssScript in allCssScripts:
dataType = re.findall('data-type=[\'"](.+?)[\'"]', cssScript)
if len(dataType) > 0:
if dataType[0] == build_config.removeFromBuild:
if removeFilesFromBuild:
html = re.sub(cssScript, "", html)
else:
replaceStr = re.sub('data-type=[\'"].+?[\'"]', '',
cssScript)
html = re.sub(cssScript, replaceStr, html)
elif dataType[0] == build_config.doNotMinify:
replaceStr = re.sub('data-type=[\'"].+?[\'"]', '', cssScript)
html = re.sub(cssScript, replaceStr, html)
else: #it's a group (we've already put out main js into a group)
cssTypesToCombine.append(dataType[0])
utils.removeDuplicates(cssTypesToCombine)
for cssDataType in cssTypesToCombine:
allRelatedScripts = re.findall(
'<link.+?data-type=[\'"]+?' + re.escape(cssDataType) +
'.+?[\'"].+?\/>', html)
if isMin:
filePath = build_config.cssOutputFolder + "/" + cssDataType + ".min.css"
html = re.sub(
allRelatedScripts[0],
'<link charset="utf-8" rel="stylesheet" href="' + filePath +
'"/>\n', html)
else:
filePath = build_config.cssOutputFolder + "/" + cssDataType + ".css"
html = re.sub(
allRelatedScripts[0],
'<link charset="utf-8" data-type="to-minify" rel="stylesheet" href="'
+ filePath + '"/>\n', html)
for script in allRelatedScripts:
html = re.sub(script, "", html)
if removeStaticPath:
html = re.sub(build_config.staticPath, "", html)
return html
def createConcatenatedFileLinksInHtml(filePath, htmlRef, removeFilesFromBuild, removeStaticPath, isMin):
html = htmlRef
#remove all local only (if removeFilesFromBuild)
# to-do: combine into single function
allJsScripts = re.findall('<script.+?[src=].+?[type=].+?[text/javascript].+?<\/script>', html)
jsTypesToCombine = []
for jsScript in allJsScripts:
dataType = re.findall('data-type=[\'"](.+?)[\'"]', jsScript)
if len(dataType) > 0:
if dataType[0] == build_config.removeFromBuild:
if removeFilesFromBuild:
html = re.sub(jsScript, "", html)
else:
replaceStr = re.sub('data-type=[\'"].+?[\'"]', '', jsScript)
html = re.sub(jsScript, replaceStr, html)
elif dataType[0] == build_config.doNotMinify:
replaceStr = re.sub('data-type=[\'"].+?[\'"]', '', jsScript)
html = re.sub(jsScript, replaceStr, html)
else: #it's a group (we've already put out main js into a group)
jsTypesToCombine.append(dataType[0])
utils.removeDuplicates(jsTypesToCombine)
for jsDataType in jsTypesToCombine:
allRelatedScripts = re.findall('<script.+?data-type=[\'"]+?' + re.escape(jsDataType) + '.+?[\'"].+?<\/script>', html)
if isMin:
filePath = build_config.jsOutputFolder + "/" + jsDataType + ".min.js"
html = re.sub(allRelatedScripts[0], '<script charset="utf-8" type="text/javascript" src="' + filePath + '"></script>\n', html)
else:
filePath = build_config.jsOutputFolder + "/" + jsDataType + ".js"
html = re.sub(allRelatedScripts[0], '<script charset="utf-8" data-type="to-minify" type="text/javascript" src="' + filePath + '"></script>\n', html)
for script in allRelatedScripts:
html = re.sub(script, "", html)
allCssScripts = re.findall('<link.+?[href=].+?[rel=\"stylesheet\"].+?[type=\"text\/css\"].+?\/>', html)
cssTypesToCombine = []
for cssScript in allCssScripts:
dataType = re.findall('data-type=[\'"](.+?)[\'"]', cssScript)
if len(dataType) > 0:
if dataType[0] == build_config.removeFromBuild:
if removeFilesFromBuild:
html = re.sub(cssScript, "", html)
else:
replaceStr = re.sub('data-type=[\'"].+?[\'"]', '', cssScript)
html = re.sub(cssScript, replaceStr, html)
elif dataType[0] == build_config.doNotMinify:
replaceStr = re.sub('data-type=[\'"].+?[\'"]', '', cssScript)
html = re.sub(cssScript, replaceStr, html)
else: #it's a group (we've already put out main js into a group)
cssTypesToCombine.append(dataType[0])
utils.removeDuplicates(cssTypesToCombine)
for cssDataType in cssTypesToCombine:
allRelatedScripts = re.findall('<link.+?data-type=[\'"]+?' + re.escape(cssDataType) + '.+?[\'"].+?\/>', html)
if isMin:
filePath = build_config.cssOutputFolder + "/" + cssDataType + ".min.css"
html = re.sub(allRelatedScripts[0], '<link charset="utf-8" rel="stylesheet" href="' + filePath + '"/>\n', html)
else:
filePath = build_config.cssOutputFolder + "/" + cssDataType + ".css"
html = re.sub(allRelatedScripts[0], '<link charset="utf-8" data-type="to-minify" rel="stylesheet" href="' + filePath + '"/>\n', html)
for script in allRelatedScripts:
html = re.sub(script, "", html)
if removeStaticPath:
html = re.sub(build_config.staticPath, "", html)
return html
archiveName="archive.json"
archive_dict = loadEvents(PATH_TO_DATA,archiveName)
os.chdir(PATH_TO_CODE)
incidentList=[]
incident_nodes=[]
for e in archive_dict: #each e is another event
incident_nodes.append(FireNode(e["latitude"],e["longitude"],"event",0,e["acq_date"],e["confidence"],e["frp"]))
incident_nodes[-1].calculateWeight(MAX_SEV,MAX_FRP,MAX_POP_DENSITY)
incidentList.append((e["latitude"],e["longitude"]))
SIZE_OF_CLUSTER=5
mean_cluster_nodes=[]
cluster_means=[]
incident_nodes=removeDuplicates(incident_nodes)
copy_incident_nodes=incident_nodes[:] # for testing
next_one=incident_nodes[-1]
sizes=[]
mean_nodes=[]
while len(incident_nodes)>=SIZE_OF_CLUSTER:
genesis_cluster=Cluster()
next_one,incident_nodes=genesis_cluster.form_cluster(incident_nodes[-1],SIZE_OF_CLUSTER,incident_nodes)
cluster_nodes=[]
for node in genesis_cluster.nodes_in_cluster:
cluster_nodes.append((node.lat,node.long))
try:
incident_nodes.remove(node)
#print("Removing")
except Exception as e:
print(str(e))
