def xmlInit(self, aDEVS):
"""
The trace functionality for XML output at initialization
Args:
aDEVS - the model that was initialised
"""
execute(
aDEVS,
easyCommand("self.tracers.saveXML", [
toStr(aDEVS.getModelFullName()), aDEVS.timeLast, "'EX'", "''",
toStr(aDEVS.state.toXML()),
toStr(aDEVS.state)
]), self.immediate)
def xmlConfluent(self, aDEVS):
"""
The trace functionality for XML output at a confluent transition
Args:
aDEVS - the model that transitioned
"""
portInfo = ""
for I in range(len(aDEVS.IPorts)):
portInfo += "<port name=\"" + aDEVS.IPorts[I].getPortName(
) + "\" category=\"I\">\n"
for j in aDEVS.peek(aDEVS.IPorts[I]):
portInfo += "<message>" + str(j) + "</message>\n</port>\n"
execute(
aDEVS,
easyCommand("self.tracers.saveXML", [
toStr(aDEVS.getModelFullName()), aDEVS.timeLast, "'EX'",
toStr(portInfo),
toStr(aDEVS.state.toXML()),
toStr(aDEVS.state)
]), self.immediate)
portInfo = ""
for I in range(len(aDEVS.OPorts)):
if aDEVS.OPorts[I] in aDEVS.myOutput:
portInfo += "<port name=\"" + aDEVS.OPorts[I].getPortName(
) + "\" category=\"O\">\n"
for j in aDEVS.myOutput[aDEVS.OPorts[I]]:
portInfo += "<message>" + str(j) + "</message>\n</port>\n"
execute(
aDEVS,
easyCommand("self.tracers.saveXML", [
toStr(aDEVS.getModelFullName()), aDEVS.timeLast, "'IN'",
toStr(portInfo),
toStr(aDEVS.state.toXML()),
toStr(aDEVS.state)
]), self.immediate)
def xmlInit(self, aDEVS):
"""
The trace functionality for XML output at initialization
Args:
aDEVS - the model that was initialised
"""
execute(aDEVS, easyCommand("self.tracers.saveXML", [toStr(aDEVS.getModelFullName()), aDEVS.timeLast, "'EX'", "''", toStr(aDEVS.state.toXML()), toStr(aDEVS.state)]), self.immediate)
def traceInit(self, aDEVS):
"""
The trace functionality for XML output at initialization
:param aDEVS: the model that transitioned
"""
runTraceAtController(self.server, self.uid, aDEVS, [toStr(aDEVS.getModelFullName()), aDEVS.timeLast, "'EX'", "''", toStr(aDEVS.state.toXML()), toStr(aDEVS.state)])
def decryptHexArray(hexArray, KEY):
assert (len(KEY) == 16), 'KEY is not the correct size: decryptHexArray'
l = []
for i in hexArray:
D_x = decryptHex(i, KEY)
D_s = toStr(D_x)
l.append(D_s)
# yield (D_s)
return ''.join(l)
def vcdExternal(self, aDEVS):
"""
The trace functionality for VCD output at an external transition
Args:
aDEVS - the model that transitioned
"""
for I in range(len(aDEVS.IPorts)):
portName = aDEVS.IPorts[I].getPortName()
signalBag = aDEVS.peek(aDEVS.IPorts[I])
if signalBag is not None:
for portSignal in signalBag:
execute(
aDEVS,
easyCommand("self.tracers.saveVCD", [
toStr(aDEVS.getModelFullName()), aDEVS.timeLast,
toStr(portName),
toStr(portSignal)
]), self.immediate)
def decryptHexToString(C_x, KEY):
assert (len(KEY) == 16), 'KEY is not the correct size: decryptHex'
l = []
for i in range(0, len(C_x), 16):
c = C_x[i: i + 16]
D_x = decryptHex(c, KEY)
D_s = toStr(D_x)
# yield (D_s)
l.append(D_s)
return ''.join(l)
def traceConfluent(self, aDEVS):
"""
The trace functionality for XML output at a confluent transition
:param aDEVS: the model that transitioned
"""
portInfo = ""
for I in range(len(aDEVS.IPorts)):
portInfo += "<port name=\""+ aDEVS.IPorts[I].getPortName()+"\" category=\"I\">\n"
for j in aDEVS.myInput[aDEVS.IPorts[I]]:
portInfo += "<message>" + str(j) + "</message>\n</port>\n"
runTraceAtController(self.server, self.uid, aDEVS, [toStr(aDEVS.getModelFullName()), aDEVS.timeLast, "'EX'", toStr(portInfo), toStr(aDEVS.state.toXML()), toStr(aDEVS.state)])
portInfo = ""
for I in range(len(aDEVS.OPorts)):
if aDEVS.OPorts[I] in aDEVS.myOutput:
portInfo += "<port name=\""+ aDEVS.OPorts[I].getPortName()+"\" category=\"O\">\n"
for j in aDEVS.myOutput[aDEVS.OPorts[I]]:
portInfo += "<message>" + str(j) + "</message>\n</port>\n"
runTraceAtController(self.server, self.uid, aDEVS, [toStr(aDEVS.getModelFullName()), aDEVS.timeLast, "'IN'", toStr(portInfo), toStr(aDEVS.state.toXML()), toStr(aDEVS.state)])
def xmlConfluent(self, aDEVS):
"""
The trace functionality for XML output at a confluent transition
Args:
aDEVS - the model that transitioned
"""
portInfo = ""
for I in range(len(aDEVS.IPorts)):
portInfo += "<port name=\""+ aDEVS.IPorts[I].getPortName()+"\" category=\"I\">\n"
for j in aDEVS.peek(aDEVS.IPorts[I]):
portInfo += "<message>" + str(j) + "</message>\n</port>\n"
execute(aDEVS, easyCommand("self.tracers.saveXML", [toStr(aDEVS.getModelFullName()), aDEVS.timeLast, "'EX'", toStr(portInfo), toStr(aDEVS.state.toXML()), toStr(aDEVS.state)]), self.immediate)
portInfo = ""
for I in range(len(aDEVS.OPorts)):
if aDEVS.OPorts[I] in aDEVS.myOutput:
portInfo += "<port name=\""+ aDEVS.OPorts[I].getPortName()+"\" category=\"O\">\n"
for j in aDEVS.myOutput[aDEVS.OPorts[I]]:
portInfo += "<message>" + str(j) + "</message>\n</port>\n"
execute(aDEVS, easyCommand("self.tracers.saveXML", [toStr(aDEVS.getModelFullName()), aDEVS.timeLast, "'IN'", toStr(portInfo), toStr(aDEVS.state.toXML()), toStr(aDEVS.state)]), self.immediate)
def vcdConfluent(self, aDEVS):
"""
The trace functionality for VCD output at a confluent transition
Args:
aDEVS - the model that transitioned
"""
name = aDEVS.getModelFullName()
for I in range(len(aDEVS.IPorts)):
portName = aDEVS.IPorts[I].getPortName()
signalBag = aDEVS.peek(aDEVS.IPorts[I])
if signalBag is not None:
for portSignal in signalBag:
execute(aDEVS, easyCommand("self.tracers.saveVCD", [toStr(aDEVS.getModelFullName()), aDEVS.timeLast, toStr(portName), toStr(portSignal)]), self.immediate)
for I in range(len(aDEVS.OPorts) ):
if aDEVS.OPorts[I] in aDEVS.myOutput:
portName = aDEVS.OPorts[I].getPortName()
signalBag = aDEVS.myOutput[aDEVS.OPorts[I]]
if signalBag is not None:
for portSignal in signalBag:
execute(aDEVS, easyCommand("self.tracers.saveVCD", [toStr(aDEVS.getModelFullName()), aDEVS.timeLast, toStr(portName), toStr(portSignal)]), self.immediate)
def traceExternal(self, aDEVS):
"""
The trace functionality for VCD output at an external transition
:param aDEVS: the model that transitioned
"""
name = toStr(aDEVS.getModelFullName())
for I in range(len(aDEVS.IPorts)):
portName = aDEVS.IPorts[I].getPortName()
signalBag = aDEVS.myInput.get(aDEVS.IPorts[I], [])
if signalBag is not None:
for portSignal in signalBag:
runTraceAtController(self.server, self.uid, aDEVS, [name, aDEVS.timeLast, toStr(portName), toStr(portSignal)])
def traceExternal(self, aDEVS):
"""
The trace functionality for Cell DEVS output at an external transition
:param aDEVS: the model that transitioned
"""
try:
runTraceAtController(self.server, self.uid, aDEVS, [
aDEVS.x, aDEVS.y, aDEVS.timeLast,
toStr(aDEVS.state.toCellState())
])
except AttributeError:
pass
def WriteBack(self, def_vals, data, final_out_file, elaborate):
if elaborate is None:
return
self.__sync.acquire()
data = [x for x in data if x is not None]
data = [x for x in data if len(x)]
# we remove the empty elements
file_name = sobstitute(elaborate.out_file_name.currValue(), self)
in_file_format = elaborate.in_file_format
res = {}
res["MIN"] = [0] * len(in_file_format)
res["MAX"] = [0] * len(in_file_format)
res["AVG"] = [0] * len(in_file_format)
res["STD"] = [0] * len(in_file_format)
if len(data):
data_ex = zip(*data)
for i in range(len(in_file_format)):
res["AVG"][i] = avg(data_ex[i])
res["MIN"][i] = min(data_ex[i])
res["MAX"][i] = max(data_ex[i])
for i in range(len(in_file_format)):
res["STD"][i] = std(data_ex[i], res["AVG"][i])
# print "MINS: " + str(res['MIN'])
# print "MAXS: " + str(res['MAX'])
# print "AVG : " + str(res['AVG'])
# print "STD : " + str(res['STD'])
vals = []
for param in elaborate.out_file_format:
if param in self.__parameters.keys():
vals.append(toStr(self.__parameters[param].currValue()))
elif (
param.startswith("AVG(")
or param.startswith("STD(")
or param.startswith("MIN(")
or param.startswith("MAX(")
) and param.endswith(")"):
param_name = param[4:-1]
assert param_name in in_file_format.values
vals.append(toStr(res[param[:3]][in_file_format.values.index(param_name)]))
self.__speedup = -1
idx = elaborate.speedup_idx.currValue()
if def_vals:
if def_vals["AVG"][idx] != 0 and res["AVG"][idx] != 0:
assert idx > 0 or idx < 0
if idx < 0:
idx = (-idx) - 1
self.__speedup = res["AVG"][idx] / def_vals["AVG"][idx]
else:
idx -= 1
self.__speedup = def_vals["AVG"][idx] / res["AVG"][idx]
vals.append(toStr(self.__speedup))
open(file_name, "a").write(",".join(vals) + "\n")
print "\n{0}".format(80 * "*")
if res["AVG"][idx] == 0:
print "* Configuration failed to execute! "
else:
print "* Average values: \n*\t%s" % ", ".join(map(lambda x: "{0:.3f}".format(x), res["AVG"]))
if self.__speedup != -1:
print "* Speedup: \n*\t{0:.5f}".format(self.__speedup)
print "{0}".format(80 * "*")
self.__sync.notify()
self.__sync.release()
return res
def WriteBack(self, def_vals, data, final_out_file, elaborate):
if elaborate is None:
return
self.__sync.acquire()
data = [x for x in data if x is not None]
data = [x for x in data if len(x)]
# we remove the empty elements
file_name = sobstitute(elaborate.out_file_name.currValue(), self)
in_file_format = elaborate.in_file_format
res = {}
res['MIN'] = [0] * len(in_file_format)
res['MAX'] = [0] * len(in_file_format)
res['AVG'] = [0] * len(in_file_format)
res['STD'] = [0] * len(in_file_format)
if len(data):
data_ex = zip(*data)
for i in range(len(in_file_format)):
res['AVG'][i] = avg(data_ex[i])
res['MIN'][i] = min(data_ex[i])
res['MAX'][i] = max(data_ex[i])
for i in range(len(in_file_format)):
res['STD'][i] = std(data_ex[i], res['AVG'][i])
# print "MINS: " + str(res['MIN'])
# print "MAXS: " + str(res['MAX'])
# print "AVG : " + str(res['AVG'])
# print "STD : " + str(res['STD'])
vals = []
for param in elaborate.out_file_format:
if param in self.__parameters.keys():
vals.append(toStr(self.__parameters[param].currValue()))
elif (param.startswith('AVG(') or param.startswith('STD(')
or param.startswith('MIN(')
or param.startswith('MAX(')) and param.endswith(')'):
param_name = param[4:-1]
assert param_name in in_file_format.values
vals.append(
toStr(res[param[:3]][in_file_format.values.index(
param_name)]))
self.__speedup = -1
idx = elaborate.speedup_idx.currValue()
if def_vals:
if def_vals['AVG'][idx] != 0 and res['AVG'][idx] != 0:
assert (idx > 0 or idx < 0)
if idx < 0:
idx = (-idx) - 1
self.__speedup = res['AVG'][idx] / def_vals['AVG'][idx]
else:
idx -= 1
self.__speedup = def_vals['AVG'][idx] / res['AVG'][idx]
vals.append(toStr(self.__speedup))
open(file_name, 'a').write(",".join(vals) + "\n")
print "\n{0}".format(80 * "*")
if res['AVG'][idx] == 0:
print "* Configuration failed to execute! "
else:
print "* Average values: \n*\t%s" % ", ".join(
map(lambda x: "{0:.3f}".format(x), res['AVG']))
if self.__speedup != -1:
print "* Speedup: \n*\t{0:.5f}".format(self.__speedup)
print "{0}".format(80 * "*")
self.__sync.notify()
self.__sync.release()
return res
def next(self):
repeatitions = 1
while repeatitions < self.__repeatitions:
self.reset()
generation_number = 1
max_fitness = None
max_generation = 0
open(self.__log_file, "a").write("@ Genetich search start: {0}\n".format(repeatitions))
while generation_number < self.__iterations or (generation_number - max_generation) < 10:
# generate a tournament
toEvaluate = [self.__pop.index(x) for x in self.__pop if not x.isEvaluated()]
# collect configurations which have not been evaluated yet
print "#{0}#".format("*" * 78)
print "# Starting new tournament"
print "# # of configurations not yet evaluated: %s" % len(toEvaluate)
tournament = set()
while len(tournament) < self.__tournament_size:
if len(toEvaluate) > 0:
n = random.randint(0, len(toEvaluate) - 1)
tournament.add(self.__pop[toEvaluate[n]])
del toEvaluate[n]
else:
tournament.add(self.__pop[random.randint(0, self.__pop_size - 1)])
# we created a tournament, now we evaluate the elements in the tournament
# print 'Population: [ {0} ]'.format( ', '.join( map(lambda x: '{0}'.format(x.isEvaluated()), self.__pop) ) )
count = 1
for element in tournament:
print "#{0}#".format("-" * 78)
print (
"# GeneticSearch: evaluating generation {0} (element {1}/{2})".format(
generation_number, count, self.__tournament_size
)
)
print "#{0}#".format("-" * 78)
print "{0}".format(element)
count += 1
if element.isEvaluated():
print "\t* Configuration already evaluated *"
continue
# set other parameters to default otherwise the runner is not able to run the configuration
added_params = []
for param in self.__config.parameters.values():
if param.name not in element.parameter_keys():
element += param
added_params.append(param)
if element.check(self.__constraints):
yield element
else:
# element failed
element.setSpeedup(-1)
for param in added_params:
element -= param
# evaluation of tournament completed, printing fitness values
tournament = sorted(tournament, key=lambda element: element.getSpeedup(), reverse=True)
print "#{0}#".format("~" * 78)
print "# Fitness values for the tournament:\n#\t[ {0} ]".format(
", ".join(map(lambda element: "{0:.5f}".format(element.getSpeedup()), tournament))
)
print "#"
tournament_max = tournament[0].getSpeedup()
if max_fitness is None or max_fitness < tournament_max:
max_fitness = tournament_max
max_generation = generation_number
# Log the outcome of this tournament
vals = []
for param in self.__log_file_format:
if param in element.parameter_keys():
vals.append(toStr(element.parameters[param].currValue()))
else:
vals.append("-")
# adding the value of the maximum fitness for this generation
vals.append("{0:.5f}".format(tournament_max))
open(self.__log_file, "a").write(",".join(vals) + "\n")
# print "##### PARENTS #####"
# print '{0}'.format(tournament[0])
# print '{0}'.format(tournament[1])
(offspring1, offspring2) = self.crossover(tournament[0], tournament[1])
# print '{0}'.format(offspring1)
# print '{0}'.format(offspring2)
if random.random() <= 0.2 or tournament[0] == tournament[1]:
# Mutation:
# in order to avoid to get stuck in a local maximal, we apply mutation
print "# Applying mutation on newly created offsprings"
for offspring in (offspring1, offspring2):
for param in offspring.parameters.values():
if random.randint(0, 1):
param.rand()
if random.randint(0, 1) and len(offspring) < len(self.__config):
print "#\t-> Adding paraleter to configuration"
# we add a new parameter to this configuration
found = False
while not found:
idx = random.randint(0, len(self.__config) - 1)
param = self.__config.parameters[self.__config.parameter_keys()[idx]]
if param.name not in offspring.parameter_keys():
offspring += param
found = True
elif len(offspring) > 1:
print "#\t-> Removing parameter to configuration"
# remove 1 parameter
idx = random.randint(0, len(offspring) - 1)
offspring -= offspring.parameters[offspring.parameter_keys()[idx]]
# Kicking out the elements with the lowest fitness from the population
# by replacing with the new offspings
self.__pop[self.__pop.index(tournament[-1])] = offspring1
self.__pop[self.__pop.index(tournament[-2])] = offspring2
print "# End tournament:\n#\tLocal maximal fitness = {0:.4f}, Global maximal = {1:.4f}".format(
tournament_max, max_fitness
)
generation_number += 1
print "#{0}#".format("~" * 78)
repeatitions += 1
raise StopIteration
def main():
# parse args
parser = argparse.ArgumentParser()
parser.add_argument('-rc', '--tweetrc', help="tweetrc file for API auth")
parser.add_argument('-ff',
'--from-file',
help="read screen names from file")
parser.add_argument('-z',
'--gzip',
dest='gzip',
action='store_true',
help="run CSV output through gzip")
parser.add_argument('usernames',
metavar="screen_name",
nargs='*',
help="list of screen names")
parser.set_defaults(gzip=False)
args = parser.parse_args()
rcPath = args.tweetrc
# verify args
if args.from_file:
logging.info("Loading users to process from file: %s" % args.from_file)
with open(args.from_file) as usersFile:
users = [line.strip(' \n') for line in usersFile.readlines()]
else:
users = args.usernames
if not len(users):
print("Please provide at least 1 screen name or file to process.")
parser.print_help()
sys.exit(1)
logging.info("Number of users to process: %d" % len(users))
# load list of fields/cols to extract
with open("USER_FIELDS") as fieldsFile:
fieldNames = [field.strip(' \n') for field in fieldsFile.readlines()]
# load twitter user conf and api
rc = TweetRc(rcPath)
api = twitter.Api(consumer_key=rc.getConsumerKey(),
consumer_secret=rc.getConsumerSecret(),
access_token_key=rc.getAccessToken(),
access_token_secret=rc.getAccessSecret(),
sleep_on_rate_limit=True,
input_encoding=None)
logging.info("Connected to API and authorized.")
# extract all given users' followers
for username in users:
logging.info("Dumping followers of %s" % username)
# create CSV file (in utf-8, including BOM)
nextCursor = -1
csv_filename = "%s-%s.csv" % (
username, datetime.datetime.now().strftime("%Y%m%d"))
# write header and rows
with open(csv_filename, 'wb') as csvFile:
startT = time.time()
numFollowers = 0
logging.info("Writing to %s" % csv_filename)
csvFile.write(u'\ufeff'.encode('utf8')) # BOM
follWriter = csv.DictWriter(csvFile,
fieldNames,
encoding='utf-8',
delimiter=',',
quotechar='|',
quoting=csv.QUOTE_MINIMAL)
follWriter.writeheader()
# fetch followers and dump to CSV
while nextCursor != 0:
try:
nextCursor, prevCursor, followers = api.GetFollowersPaged(
screen_name=username, cursor=nextCursor)
except:
traceback.print_exc(file=sys.stderr)
logging.exception(
"Error obtaining next page of followers (w/ nextCursor=%d)"
% nextCursor)
time.sleep(SLEEP_ON_ERROR_MINS * 60)
else:
for u in followers:
newRow = {
fld: toStr(getattr(u, fld))
for fld in fieldNames
}
follWriter.writerow(newRow)
numFollowers += 1
endT = time.time()
logging.info("%d followers of %s dumped to %s in %f" %
(numFollowers, username, csv_filename, endT - startT))
# (optionally) gzip and delete CSV file
if args.gzip:
with open(csv_filename, 'rb') as csvFile:
gz_filename = csv_filename + '.gz'
with gzip.open(gz_filename, 'wb') as gzFile:
shutil.copyfileobj(csvFile, gzFile)
logging.info("CSV compressed to %s" % gz_filename)
os.remove(csv_filename)
logging.info("Plain file %s deleted" % csv_filename)
def traceConfluent(self, aDEVS):
"""
The trace functionality for Cell DEVS output at a confluent transition
:param aDEVS: the model that transitioned
"""
try:
runTraceAtController(self.server, self.uid, aDEVS, [aDEVS.x, aDEVS.y, aDEVS.timeLast, toStr(aDEVS.state.toCellState())])
except AttributeError as e:
print(e)
pass
def traceInit(self, aDEVS):
"""
The trace functionality for Cell DEVS output at initialisation
:param aDEVS: the model that was initialised
"""
try:
runTraceAtController(self.server, self.uid, aDEVS, [aDEVS.x, aDEVS.y, aDEVS.timeLast, toStr(aDEVS.state.toCellState())])
except AttributeError:
pass
