Exemple #1
0
 def xmlInit(self, aDEVS):
     """
     The trace functionality for XML output at initialization
     Args:
         aDEVS - the model that was initialised
     """
     execute(
         aDEVS,
         easyCommand("self.tracers.saveXML", [
             toStr(aDEVS.getModelFullName()), aDEVS.timeLast, "'EX'", "''",
             toStr(aDEVS.state.toXML()),
             toStr(aDEVS.state)
         ]), self.immediate)
Exemple #2
0
 def xmlConfluent(self, aDEVS):
     """
     The trace functionality for XML output at a confluent transition
     Args:
         aDEVS - the model that transitioned
     """
     portInfo = ""
     for I in range(len(aDEVS.IPorts)):
         portInfo += "<port name=\"" + aDEVS.IPorts[I].getPortName(
         ) + "\" category=\"I\">\n"
         for j in aDEVS.peek(aDEVS.IPorts[I]):
             portInfo += "<message>" + str(j) + "</message>\n</port>\n"
     execute(
         aDEVS,
         easyCommand("self.tracers.saveXML", [
             toStr(aDEVS.getModelFullName()), aDEVS.timeLast, "'EX'",
             toStr(portInfo),
             toStr(aDEVS.state.toXML()),
             toStr(aDEVS.state)
         ]), self.immediate)
     portInfo = ""
     for I in range(len(aDEVS.OPorts)):
         if aDEVS.OPorts[I] in aDEVS.myOutput:
             portInfo += "<port name=\"" + aDEVS.OPorts[I].getPortName(
             ) + "\" category=\"O\">\n"
             for j in aDEVS.myOutput[aDEVS.OPorts[I]]:
                 portInfo += "<message>" + str(j) + "</message>\n</port>\n"
     execute(
         aDEVS,
         easyCommand("self.tracers.saveXML", [
             toStr(aDEVS.getModelFullName()), aDEVS.timeLast, "'IN'",
             toStr(portInfo),
             toStr(aDEVS.state.toXML()),
             toStr(aDEVS.state)
         ]), self.immediate)
 def xmlInit(self, aDEVS):
     """
     The trace functionality for XML output at initialization
     Args:
         aDEVS - the model that was initialised
     """
     execute(aDEVS, easyCommand("self.tracers.saveXML", [toStr(aDEVS.getModelFullName()), aDEVS.timeLast, "'EX'", "''", toStr(aDEVS.state.toXML()), toStr(aDEVS.state)]), self.immediate)
Exemple #4
0
    def traceInit(self, aDEVS):
        """
        The trace functionality for XML output at initialization

        :param aDEVS: the model that transitioned
        """
        runTraceAtController(self.server, self.uid, aDEVS, [toStr(aDEVS.getModelFullName()), aDEVS.timeLast, "'EX'", "''", toStr(aDEVS.state.toXML()), toStr(aDEVS.state)])
Exemple #5
0
def decryptHexArray(hexArray, KEY):
    assert (len(KEY) == 16), 'KEY is not the correct size: decryptHexArray'
    l = []
    for i in hexArray:
        D_x = decryptHex(i, KEY)
        D_s = toStr(D_x)
        l.append(D_s)
        # yield (D_s)
    return ''.join(l)
Exemple #6
0
 def vcdExternal(self, aDEVS):
     """
     The trace functionality for VCD output at an external transition
     Args:
         aDEVS - the model that transitioned
     """
     for I in range(len(aDEVS.IPorts)):
         portName = aDEVS.IPorts[I].getPortName()
         signalBag = aDEVS.peek(aDEVS.IPorts[I])
         if signalBag is not None:
             for portSignal in signalBag:
                 execute(
                     aDEVS,
                     easyCommand("self.tracers.saveVCD", [
                         toStr(aDEVS.getModelFullName()), aDEVS.timeLast,
                         toStr(portName),
                         toStr(portSignal)
                     ]), self.immediate)
Exemple #7
0
def decryptHexToString(C_x, KEY):
    assert (len(KEY) == 16), 'KEY is not the correct size: decryptHex'
    l = []
    for i in range(0, len(C_x), 16):
        c = C_x[i: i + 16]
        D_x = decryptHex(c, KEY)
        D_s = toStr(D_x)
        # yield (D_s)
        l.append(D_s)
    return ''.join(l)
Exemple #8
0
    def traceConfluent(self, aDEVS):
        """
        The trace functionality for XML output at a confluent transition

        :param aDEVS: the model that transitioned
        """
        portInfo = ""
        for I in range(len(aDEVS.IPorts)):
            portInfo += "<port name=\""+ aDEVS.IPorts[I].getPortName()+"\" category=\"I\">\n"
            for j in aDEVS.myInput[aDEVS.IPorts[I]]:
                portInfo += "<message>" + str(j) + "</message>\n</port>\n"
        runTraceAtController(self.server, self.uid, aDEVS, [toStr(aDEVS.getModelFullName()), aDEVS.timeLast, "'EX'", toStr(portInfo), toStr(aDEVS.state.toXML()), toStr(aDEVS.state)])
        portInfo = ""
        for I in range(len(aDEVS.OPorts)):
            if aDEVS.OPorts[I] in aDEVS.myOutput:
                portInfo += "<port name=\""+ aDEVS.OPorts[I].getPortName()+"\" category=\"O\">\n"
                for j in aDEVS.myOutput[aDEVS.OPorts[I]]:
                    portInfo += "<message>" + str(j) + "</message>\n</port>\n"
        runTraceAtController(self.server, self.uid, aDEVS, [toStr(aDEVS.getModelFullName()), aDEVS.timeLast, "'IN'", toStr(portInfo), toStr(aDEVS.state.toXML()), toStr(aDEVS.state)])
 def xmlConfluent(self, aDEVS):
     """
     The trace functionality for XML output at a confluent transition
     Args:
         aDEVS - the model that transitioned
     """
     portInfo = ""
     for I in range(len(aDEVS.IPorts)):
         portInfo += "<port name=\""+ aDEVS.IPorts[I].getPortName()+"\" category=\"I\">\n"
         for j in aDEVS.peek(aDEVS.IPorts[I]):
             portInfo += "<message>" + str(j) + "</message>\n</port>\n"
     execute(aDEVS, easyCommand("self.tracers.saveXML", [toStr(aDEVS.getModelFullName()), aDEVS.timeLast, "'EX'", toStr(portInfo), toStr(aDEVS.state.toXML()), toStr(aDEVS.state)]), self.immediate)
     portInfo = ""
     for I in range(len(aDEVS.OPorts)):
         if aDEVS.OPorts[I] in aDEVS.myOutput:
             portInfo += "<port name=\""+ aDEVS.OPorts[I].getPortName()+"\" category=\"O\">\n"
             for j in aDEVS.myOutput[aDEVS.OPorts[I]]:
                 portInfo += "<message>" + str(j) + "</message>\n</port>\n"
     execute(aDEVS, easyCommand("self.tracers.saveXML", [toStr(aDEVS.getModelFullName()), aDEVS.timeLast, "'IN'", toStr(portInfo), toStr(aDEVS.state.toXML()), toStr(aDEVS.state)]), self.immediate)
Exemple #10
0
 def vcdConfluent(self, aDEVS):
     """
     The trace functionality for VCD output at a confluent transition
     Args:
         aDEVS - the model that transitioned
     """
     name = aDEVS.getModelFullName()
     for I in range(len(aDEVS.IPorts)):
         portName = aDEVS.IPorts[I].getPortName()
         signalBag = aDEVS.peek(aDEVS.IPorts[I])
         if signalBag is not None:
             for portSignal in signalBag:
                 execute(aDEVS, easyCommand("self.tracers.saveVCD", [toStr(aDEVS.getModelFullName()), aDEVS.timeLast, toStr(portName), toStr(portSignal)]), self.immediate)
     for I in range(len(aDEVS.OPorts) ):
         if aDEVS.OPorts[I] in aDEVS.myOutput:
             portName = aDEVS.OPorts[I].getPortName()
             signalBag = aDEVS.myOutput[aDEVS.OPorts[I]]
             if signalBag is not None:
                 for portSignal in signalBag:
                     execute(aDEVS, easyCommand("self.tracers.saveVCD", [toStr(aDEVS.getModelFullName()), aDEVS.timeLast, toStr(portName), toStr(portSignal)]), self.immediate)
Exemple #11
0
    def traceExternal(self, aDEVS):
        """
        The trace functionality for VCD output at an external transition

        :param aDEVS: the model that transitioned
        """
        name = toStr(aDEVS.getModelFullName())
        for I in range(len(aDEVS.IPorts)):
            portName = aDEVS.IPorts[I].getPortName()
            signalBag = aDEVS.myInput.get(aDEVS.IPorts[I], [])
            if signalBag is not None:
                for portSignal in signalBag:
                    runTraceAtController(self.server, self.uid, aDEVS, [name, aDEVS.timeLast, toStr(portName), toStr(portSignal)])
Exemple #12
0
    def traceExternal(self, aDEVS):
        """
        The trace functionality for Cell DEVS output at an external transition

        :param aDEVS: the model that transitioned
        """
        try:
            runTraceAtController(self.server, self.uid, aDEVS, [
                aDEVS.x, aDEVS.y, aDEVS.timeLast,
                toStr(aDEVS.state.toCellState())
            ])
        except AttributeError:
            pass
    def WriteBack(self, def_vals, data, final_out_file, elaborate):

        if elaborate is None:
            return

        self.__sync.acquire()
        data = [x for x in data if x is not None]
        data = [x for x in data if len(x)]
        # we remove the empty elements

        file_name = sobstitute(elaborate.out_file_name.currValue(), self)
        in_file_format = elaborate.in_file_format

        res = {}
        res["MIN"] = [0] * len(in_file_format)
        res["MAX"] = [0] * len(in_file_format)
        res["AVG"] = [0] * len(in_file_format)
        res["STD"] = [0] * len(in_file_format)

        if len(data):
            data_ex = zip(*data)
            for i in range(len(in_file_format)):
                res["AVG"][i] = avg(data_ex[i])
                res["MIN"][i] = min(data_ex[i])
                res["MAX"][i] = max(data_ex[i])

            for i in range(len(in_file_format)):
                res["STD"][i] = std(data_ex[i], res["AVG"][i])

            # print "MINS: " + str(res['MIN'])
            # print "MAXS: " + str(res['MAX'])
            # print "AVG : " + str(res['AVG'])
            # print "STD : " + str(res['STD'])

        vals = []
        for param in elaborate.out_file_format:
            if param in self.__parameters.keys():
                vals.append(toStr(self.__parameters[param].currValue()))
            elif (
                param.startswith("AVG(")
                or param.startswith("STD(")
                or param.startswith("MIN(")
                or param.startswith("MAX(")
            ) and param.endswith(")"):
                param_name = param[4:-1]
                assert param_name in in_file_format.values
                vals.append(toStr(res[param[:3]][in_file_format.values.index(param_name)]))

        self.__speedup = -1
        idx = elaborate.speedup_idx.currValue()
        if def_vals:
            if def_vals["AVG"][idx] != 0 and res["AVG"][idx] != 0:
                assert idx > 0 or idx < 0
                if idx < 0:
                    idx = (-idx) - 1
                    self.__speedup = res["AVG"][idx] / def_vals["AVG"][idx]
                else:
                    idx -= 1
                    self.__speedup = def_vals["AVG"][idx] / res["AVG"][idx]

            vals.append(toStr(self.__speedup))

        open(file_name, "a").write(",".join(vals) + "\n")

        print "\n{0}".format(80 * "*")
        if res["AVG"][idx] == 0:
            print "* Configuration failed to execute! "
        else:
            print "* Average values: \n*\t%s" % ", ".join(map(lambda x: "{0:.3f}".format(x), res["AVG"]))
            if self.__speedup != -1:
                print "* Speedup: \n*\t{0:.5f}".format(self.__speedup)
        print "{0}".format(80 * "*")

        self.__sync.notify()
        self.__sync.release()
        return res
Exemple #14
0
    def WriteBack(self, def_vals, data, final_out_file, elaborate):

        if elaborate is None:
            return

        self.__sync.acquire()
        data = [x for x in data if x is not None]
        data = [x for x in data if len(x)]
        # we remove the empty elements

        file_name = sobstitute(elaborate.out_file_name.currValue(), self)
        in_file_format = elaborate.in_file_format

        res = {}
        res['MIN'] = [0] * len(in_file_format)
        res['MAX'] = [0] * len(in_file_format)
        res['AVG'] = [0] * len(in_file_format)
        res['STD'] = [0] * len(in_file_format)

        if len(data):
            data_ex = zip(*data)
            for i in range(len(in_file_format)):
                res['AVG'][i] = avg(data_ex[i])
                res['MIN'][i] = min(data_ex[i])
                res['MAX'][i] = max(data_ex[i])

            for i in range(len(in_file_format)):
                res['STD'][i] = std(data_ex[i], res['AVG'][i])

            # print "MINS: " + str(res['MIN'])
            # print "MAXS: " + str(res['MAX'])
            # print "AVG : " + str(res['AVG'])
            # print "STD : " + str(res['STD'])

        vals = []
        for param in elaborate.out_file_format:
            if param in self.__parameters.keys():
                vals.append(toStr(self.__parameters[param].currValue()))
            elif (param.startswith('AVG(') or param.startswith('STD(')
                  or param.startswith('MIN(')
                  or param.startswith('MAX(')) and param.endswith(')'):
                param_name = param[4:-1]
                assert param_name in in_file_format.values
                vals.append(
                    toStr(res[param[:3]][in_file_format.values.index(
                        param_name)]))

        self.__speedup = -1
        idx = elaborate.speedup_idx.currValue()
        if def_vals:
            if def_vals['AVG'][idx] != 0 and res['AVG'][idx] != 0:
                assert (idx > 0 or idx < 0)
                if idx < 0:
                    idx = (-idx) - 1
                    self.__speedup = res['AVG'][idx] / def_vals['AVG'][idx]
                else:
                    idx -= 1
                    self.__speedup = def_vals['AVG'][idx] / res['AVG'][idx]

            vals.append(toStr(self.__speedup))

        open(file_name, 'a').write(",".join(vals) + "\n")

        print "\n{0}".format(80 * "*")
        if res['AVG'][idx] == 0:
            print "* Configuration failed to execute! "
        else:
            print "* Average values: \n*\t%s" % ", ".join(
                map(lambda x: "{0:.3f}".format(x), res['AVG']))
            if self.__speedup != -1:
                print "* Speedup: \n*\t{0:.5f}".format(self.__speedup)
        print "{0}".format(80 * "*")

        self.__sync.notify()
        self.__sync.release()
        return res
Exemple #15
0
    def next(self):
        repeatitions = 1
        while repeatitions < self.__repeatitions:
            self.reset()
            generation_number = 1
            max_fitness = None
            max_generation = 0
            open(self.__log_file, "a").write("@ Genetich search start: {0}\n".format(repeatitions))
            while generation_number < self.__iterations or (generation_number - max_generation) < 10:
                # generate a tournament

                toEvaluate = [self.__pop.index(x) for x in self.__pop if not x.isEvaluated()]
                # collect configurations which have not been evaluated yet
                print "#{0}#".format("*" * 78)
                print "#    Starting new tournament"
                print "#       # of configurations not yet evaluated: %s" % len(toEvaluate)

                tournament = set()
                while len(tournament) < self.__tournament_size:
                    if len(toEvaluate) > 0:
                        n = random.randint(0, len(toEvaluate) - 1)
                        tournament.add(self.__pop[toEvaluate[n]])
                        del toEvaluate[n]
                    else:
                        tournament.add(self.__pop[random.randint(0, self.__pop_size - 1)])
                # we created a tournament, now we evaluate the elements in the tournament
                # print 'Population: [ {0} ]'.format( ', '.join( map(lambda x: '{0}'.format(x.isEvaluated()), self.__pop) ) )
                count = 1

                for element in tournament:
                    print "#{0}#".format("-" * 78)
                    print (
                        "#    GeneticSearch: evaluating generation {0} (element {1}/{2})".format(
                            generation_number, count, self.__tournament_size
                        )
                    )
                    print "#{0}#".format("-" * 78)

                    print "{0}".format(element)

                    count += 1

                    if element.isEvaluated():
                        print "\t* Configuration already evaluated *"
                        continue

                    # set other parameters to default otherwise the runner is not able to run the configuration
                    added_params = []
                    for param in self.__config.parameters.values():
                        if param.name not in element.parameter_keys():
                            element += param
                            added_params.append(param)

                    if element.check(self.__constraints):
                        yield element
                    else:
                        # element failed
                        element.setSpeedup(-1)

                    for param in added_params:
                        element -= param

                # evaluation of tournament completed, printing fitness values
                tournament = sorted(tournament, key=lambda element: element.getSpeedup(), reverse=True)
                print "#{0}#".format("~" * 78)
                print "#    Fitness values for the tournament:\n#\t[ {0} ]".format(
                    ", ".join(map(lambda element: "{0:.5f}".format(element.getSpeedup()), tournament))
                )
                print "#"
                tournament_max = tournament[0].getSpeedup()
                if max_fitness is None or max_fitness < tournament_max:
                    max_fitness = tournament_max
                    max_generation = generation_number

                # Log the outcome of this tournament
                vals = []
                for param in self.__log_file_format:
                    if param in element.parameter_keys():
                        vals.append(toStr(element.parameters[param].currValue()))
                    else:
                        vals.append("-")
                # adding the value of the maximum fitness for this generation
                vals.append("{0:.5f}".format(tournament_max))
                open(self.__log_file, "a").write(",".join(vals) + "\n")

                # print "##### PARENTS #####"
                # print '{0}'.format(tournament[0])
                # print '{0}'.format(tournament[1])
                (offspring1, offspring2) = self.crossover(tournament[0], tournament[1])
                # print '{0}'.format(offspring1)
                # print '{0}'.format(offspring2)
                if random.random() <= 0.2 or tournament[0] == tournament[1]:
                    # Mutation:
                    # in order to avoid to get stuck in a local maximal, we apply mutation
                    print "#    Applying mutation on newly created offsprings"
                    for offspring in (offspring1, offspring2):
                        for param in offspring.parameters.values():
                            if random.randint(0, 1):
                                param.rand()

                        if random.randint(0, 1) and len(offspring) < len(self.__config):
                            print "#\t-> Adding paraleter to configuration"
                            # we add a new parameter to this configuration
                            found = False
                            while not found:
                                idx = random.randint(0, len(self.__config) - 1)
                                param = self.__config.parameters[self.__config.parameter_keys()[idx]]
                                if param.name not in offspring.parameter_keys():
                                    offspring += param
                                    found = True
                        elif len(offspring) > 1:
                            print "#\t-> Removing parameter to configuration"
                            # remove 1 parameter
                            idx = random.randint(0, len(offspring) - 1)
                            offspring -= offspring.parameters[offspring.parameter_keys()[idx]]

                # Kicking out the elements with the lowest fitness from the population
                # by replacing with the new offspings
                self.__pop[self.__pop.index(tournament[-1])] = offspring1
                self.__pop[self.__pop.index(tournament[-2])] = offspring2

                print "#    End tournament:\n#\tLocal maximal fitness = {0:.4f}, Global maximal = {1:.4f}".format(
                    tournament_max, max_fitness
                )
                generation_number += 1

                print "#{0}#".format("~" * 78)

            repeatitions += 1

        raise StopIteration
def main():

    # parse args
    parser = argparse.ArgumentParser()
    parser.add_argument('-rc', '--tweetrc', help="tweetrc file for API auth")
    parser.add_argument('-ff',
                        '--from-file',
                        help="read screen names from file")
    parser.add_argument('-z',
                        '--gzip',
                        dest='gzip',
                        action='store_true',
                        help="run CSV output through gzip")
    parser.add_argument('usernames',
                        metavar="screen_name",
                        nargs='*',
                        help="list of screen names")
    parser.set_defaults(gzip=False)
    args = parser.parse_args()
    rcPath = args.tweetrc

    # verify args
    if args.from_file:
        logging.info("Loading users to process from file: %s" % args.from_file)
        with open(args.from_file) as usersFile:
            users = [line.strip(' \n') for line in usersFile.readlines()]
    else:
        users = args.usernames

    if not len(users):
        print("Please provide at least 1 screen name or file to process.")
        parser.print_help()
        sys.exit(1)

    logging.info("Number of users to process: %d" % len(users))

    # load list of fields/cols to extract
    with open("USER_FIELDS") as fieldsFile:
        fieldNames = [field.strip(' \n') for field in fieldsFile.readlines()]

    # load twitter user conf and api
    rc = TweetRc(rcPath)
    api = twitter.Api(consumer_key=rc.getConsumerKey(),
                      consumer_secret=rc.getConsumerSecret(),
                      access_token_key=rc.getAccessToken(),
                      access_token_secret=rc.getAccessSecret(),
                      sleep_on_rate_limit=True,
                      input_encoding=None)
    logging.info("Connected to API and authorized.")

    # extract all given users' followers
    for username in users:
        logging.info("Dumping followers of %s" % username)
        # create CSV file (in utf-8, including BOM)
        nextCursor = -1
        csv_filename = "%s-%s.csv" % (
            username, datetime.datetime.now().strftime("%Y%m%d"))
        # write header and rows
        with open(csv_filename, 'wb') as csvFile:
            startT = time.time()
            numFollowers = 0
            logging.info("Writing to %s" % csv_filename)
            csvFile.write(u'\ufeff'.encode('utf8'))  # BOM
            follWriter = csv.DictWriter(csvFile,
                                        fieldNames,
                                        encoding='utf-8',
                                        delimiter=',',
                                        quotechar='|',
                                        quoting=csv.QUOTE_MINIMAL)
            follWriter.writeheader()
            # fetch followers and dump to CSV
            while nextCursor != 0:
                try:
                    nextCursor, prevCursor, followers = api.GetFollowersPaged(
                        screen_name=username, cursor=nextCursor)
                except:
                    traceback.print_exc(file=sys.stderr)
                    logging.exception(
                        "Error obtaining next page of followers (w/ nextCursor=%d)"
                        % nextCursor)
                    time.sleep(SLEEP_ON_ERROR_MINS * 60)
                else:
                    for u in followers:
                        newRow = {
                            fld: toStr(getattr(u, fld))
                            for fld in fieldNames
                        }
                        follWriter.writerow(newRow)
                        numFollowers += 1
            endT = time.time()
            logging.info("%d followers of %s dumped to %s in %f" %
                         (numFollowers, username, csv_filename, endT - startT))

        # (optionally) gzip and delete CSV file
        if args.gzip:
            with open(csv_filename, 'rb') as csvFile:
                gz_filename = csv_filename + '.gz'
                with gzip.open(gz_filename, 'wb') as gzFile:
                    shutil.copyfileobj(csvFile, gzFile)
                    logging.info("CSV compressed to %s" % gz_filename)
            os.remove(csv_filename)
            logging.info("Plain file %s deleted" % csv_filename)
    def traceConfluent(self, aDEVS):
        """
        The trace functionality for Cell DEVS output at a confluent transition

        :param aDEVS: the model that transitioned
        """
        try:
            runTraceAtController(self.server, self.uid, aDEVS, [aDEVS.x, aDEVS.y, aDEVS.timeLast, toStr(aDEVS.state.toCellState())])
        except AttributeError as e:
            print(e)
            pass
    def traceInit(self, aDEVS):
        """
        The trace functionality for Cell DEVS output at initialisation

        :param aDEVS: the model that was initialised
        """
        try:
            runTraceAtController(self.server, self.uid, aDEVS, [aDEVS.x, aDEVS.y, aDEVS.timeLast, toStr(aDEVS.state.toCellState())])
        except AttributeError:
            pass