def execution(result): # download the raw data update(date_begin,date_end) #### Modify folderpath #### folderpath = '../Download/BusLocation_09_05_2017' aggregate(folderpath) # ridership analysis database="ridership" user="******" password="******" host="localhost" port="5432" ridership(database,user,password,host,port) # round trip time folderpath = '../output' database="RTT" RTT_upload(database,user,password,host,port) run_RTT(folderpath) # interarrival database="inter_arrival" inter_arrival_upload(database,user,password,host,port) run_interarrival(folderpath) return
def main():
print("Start Main")
env_params = init_env_params()
B_new, env_params_updated = update_environment(env_params, B_init)
k_prime, _ = init_kprime_kcross(env_params_updated)
print(np.sum(env_params_updated['id_shocks']))
print(np.sum(env_params_updated['agg_shocks']))
diff_B = dif_B
while diff_B > criter_B:
k_prime_new, c = individual_optimization(k_prime, env_params_updated)
km_series, k_cross_new = aggregate(k_prime_new, env_params_updated)
B_new, env_params_updated = update_environment(env_params,
B_new,
k_cross_new=k_cross_new,
km_ts=km_series)
print("diffB", env_params_updated['diffB'])
diff_B = env_params_updated['diffB']
k_prime = k_prime_new
plot_accuracy(km_series, env_params_updated['agg_shocks'], B_new)
plot_policy(k_prime_new, km_series, env_params_updated)
plot_lorenz(k_cross_new)
def publish(self, src, dest): srcns = [] fullsrc = self.dw.resolve(src, [], srcns) # doc = self.dw.getpage(src, [], srcns) destns = [] self.dw.resolve(dest, [], destns) toc = [ ' - [[%s]]' % fullsrc ] doc, chapters = aggregate(self.dw, toc, srcns) # print(len(doc)) # print(chapters) newdoc = [] for line in doc: re1line = wiki.rx_link.sub(lambda m: self.resolve_link(srcns, destns, m), line) re2line = wiki.rx_image.sub(lambda m: self.resolve_image(srcns, destns, m), re1line) newdoc.append(re2line) self.dw.putpage(newdoc, dest, summary='publish page')
def publish(self, src, dest):
srcns = []
fullsrc = self.dw.resolve(src, [], srcns)
# doc = self.dw.getpage(src, [], srcns)
destns = []
self.dw.resolve(dest, [], destns)
toc = [' - [[%s]]' % fullsrc]
doc, chapters = aggregate(self.dw, toc, srcns)
# print(len(doc))
# print(chapters)
newdoc = []
for line in doc:
re1line = Wiki.rx_link.sub(
lambda m: self.resolve_link(srcns, destns, m), line)
re2line = Wiki.rx_image.sub(
lambda m: self.resolve_image(srcns, destns, m), re1line)
newdoc.append(re2line)
self.dw.putpage(newdoc, dest, summary='publish page')
def perform(self, fidoc):
dw = fidoc.get_wiki()
logging.info("Loading table of contents %s ..." % self.tocpage)
tocns = []
toc = dw.getpage(self.tocpage, pagens = tocns)
if toc is None:
logging.error("Table of contents %s not found." % self.tocpage)
return False
logging.info("Aggregating pages ...")
doc, chapters = aggregate(dw, toc, tocns, self.embedwikilinks)
logging.info("Flushing generated content to page %s ..." % self.outpage)
res = dw.putpage(doc, self.outpage)
# print(res)
# locks = dw.lockpage(self.outpage)
# logging.info("Locks: %s" % locks)
return res
def perform(self, fidoc):
dw = fidoc.get_wiki()
logging.info("Loading table of contents %s ..." % self.tocpage)
tocns = []
toc = dw.getpage(self.tocpage, pagens=tocns)
if toc is None:
logging.error("Table of contents %s not found." % self.tocpage)
return False
logging.info("Aggregating pages ...")
doc, chapters = aggregate(dw, toc, tocns, self.embedwikilinks)
logging.info("Flushing generated content to page %s ..." %
self.outpage)
res = dw.putpage(doc, self.outpage)
# print(res)
# locks = dw.lockpage(self.outpage)
# logging.info("Locks: %s" % locks)
return res
def generatedoc(templatefile, generatefile, fidoc, tocpage, aggregatefile=None, chapterfile=None, injectrefs=False, ignorepagelinks=[], imagepath = "_media/"):
dw = fidoc.get_wiki()
document = docxwrapper(templatefile, imagepath)
tocns = []
toc = dw.getpage(tocpage, pagens = tocns)
if toc is None:
logging.fatal("Table of Contents %s not found." % tocpage)
doc, chapters = aggregate(dw, toc, tocns, showwikiurl = injectrefs)
# print()
# rx_pub_ns = re.compile('^:ficontent:((socialtv|smartcity|gaming|common):.*|deliverables:d[0-9]+|fiware:ge_usage)$')
# pub = wikipublisher(dw, mirror.public_pages())
pub = fidoc.get_publisher()
wp = wikiprocessor(document, dw, tocns, chapters, pub, imagepath, ignorepagelinks)
if aggregatefile is not None:
with open(aggregatefile, "w") as fo:
fo.writelines([l + '\n' for l in doc])
if chapterfile is not None:
import json
with open(chapterfile, 'w') as cf:
json.dump(chapters, cf, sort_keys = False, indent = 4)
# process aggregated document
collectedlines = []
lastline = None
for line in doc:
# print(line)
# line = unicode(line, "utf-8")
linetype = None
line = wp.replacelinks(line)
for key, rx in rx_line.items():
result = rx.match(line)
if result is not None:
linetype = key
break
# print "linetype1:", linetype
if linetype is None:
linetype = "text"
if linetype == "endcode":
linetype = "empty"
elif lastline == "code":
linetype = "code"
if linetype == "endgraph":
linetype = "empty"
elif lastline == "graph":
linetype = "graph"
# print "linetype2:", linetype
if lastline != linetype:
try:
wp.processlines(lastline, collectedlines)
collectedlines = []
except (Exception,UnicodeEncodeError) as e:
print("Problem with lines!")
for l in collectedlines:
print(l)
print()
print(e)
raise e
break
# print "Process lines:", lastline, " - ", len(collectedlines)
lastline = linetype
collectedlines.append(line)
document.generate(generatefile)
def _processAutostackMetricRequests(self, engine, requests, modelSwapper):
""" Execute autostack metric requests, aggregate and stream
collected metric data
:param engine: SQLAlchemy engine object
:type engine: sqlalchemy.engine.Engine
:param requests: sequence of AutostackMetricRequest objects
:param modelSwapper: Model Swapper
"""
# Start collecting requested metric data
collectionIter = self._metricGetter.collectMetricData(requests)
# Aggregate each collection and dispatch to app MetricStreamer
for metricCollection in collectionIter:
request = requests[metricCollection.refID]
metricObj = request.metric
data = None
if metricCollection.slices:
aggregationFn = getAggregationFn(metricObj)
if aggregationFn:
data = aggregate(metricCollection.slices,
aggregationFn=aggregationFn)
else:
data = aggregate(metricCollection.slices)
try:
with engine.connect() as conn:
repository.retryOnTransientErrors(repository.setMetricLastTimestamp)(
conn, metricObj.uid, metricCollection.nextMetricTime)
except ObjectNotFoundError:
self._log.warning("Processing autostack data collection results for "
"unknown model=%s (model deleted?)", metricObj.uid)
continue
if data:
try:
self.metricStreamer.streamMetricData(data,
metricID=metricObj.uid,
modelSwapper=modelSwapper)
except ObjectNotFoundError:
# We expect that the model exists but in the odd case that it has
# already been deleted we don't want to crash the process.
self._log.info("Metric not found when adding data. metric=%s" %
metricObj.uid)
self._log.debug(
"{TAG:APP.AGG.DATA.PUB} Published numItems=%d for metric=%s;"
"timeRange=[%sZ-%sZ]; headTS=%sZ; tailTS=%sZ",
len(data), getMetricLogPrefix(metricObj),
metricCollection.timeRange.start.isoformat(),
metricCollection.timeRange.end.isoformat(),
data[0][0].isoformat(), data[-1][0].isoformat())
else:
self._log.info(
"{TAG:APP.AGG.DATA.NONE} No data for metric=%s;"
"timeRange=[%sZ-%sZ]", getMetricLogPrefix(metricObj),
metricCollection.timeRange.start.isoformat(),
metricCollection.timeRange.end.isoformat())
def generatedoc(templatefile,
generatefile,
fidoc,
tocpage,
aggregatefile=None,
chapterfile=None,
injectrefs=False,
ignorepagelinks=[],
imagepath="_media/"):
dw = fidoc.get_wiki()
document = docxwrapper(templatefile, imagepath)
tocns = []
toc = dw.getpage(tocpage, pagens=tocns)
if toc is None:
logging.fatal("Table of Contents %s not found." % tocpage)
doc, chapters = aggregate(dw, toc, tocns, showwikiurl=injectrefs)
# print()
# rx_pub_ns = re.compile('^:ficontent:((socialtv|smartcity|gaming|common):.*|deliverables:d[0-9]+|fiware:ge_usage)$')
# pub = wikipublisher(dw, mirror.public_pages())
pub = fidoc.get_publisher()
wp = wikiprocessor(document, dw, tocns, chapters, pub, imagepath,
ignorepagelinks)
if aggregatefile is not None:
with open(aggregatefile, "w") as fo:
fo.writelines([l + '\n' for l in doc])
if chapterfile is not None:
import json
with open(chapterfile, 'w') as cf:
json.dump(chapters, cf, sort_keys=False, indent=4)
# process aggregated document
collectedlines = []
lastline = None
for line in doc:
# print(line)
# line = unicode(line, "utf-8")
linetype = None
line = wp.replacelinks(line)
for key, rx in rx_line.items():
result = rx.match(line)
if result is not None:
linetype = key
break
# print "linetype1:", linetype
if linetype is None:
linetype = "text"
if linetype == "endcode":
linetype = "empty"
elif lastline == "code":
linetype = "code"
if linetype == "endgraph":
linetype = "empty"
elif lastline == "graph":
linetype = "graph"
# print "linetype2:", linetype
if lastline != linetype:
try:
wp.processlines(lastline, collectedlines)
collectedlines = []
except (Exception, UnicodeEncodeError) as e:
print("Problem with lines!")
for l in collectedlines:
print(l)
print()
print(e)
raise e
break
# print "Process lines:", lastline, " - ", len(collectedlines)
lastline = linetype
collectedlines.append(line)
document.generate(generatefile)
