def info_extraction(link, cats):
response = requests.get(link + '/tabs/description')
sleep(randint(0, 4))
while response.status_code != 200:
response = requests.get(link + '/tabs/description')
sleep(randint(0, 4))
soup = BeautifulSoup(response.content, 'html5lib')
title = soup.find('h1', attrs={'data-test-id': 'project-title'}).text
project_holder = URL + soup.find('div', class_='intro__StyledOwner-sc-1wfo5yh-8 iEVesU').a['href']
desc = soup.find('head').find('meta', attrs={'name': 'description'})['content']
completion_rate = int(soup.find('div', class_='progress__StyledText-ycznm1-1').text[:3])
prompt = soup.find_all('div', class_='clfvNf')
sub_prompt = soup.find_all('div', class_='gDdVEG')
nb_contrib = int(prompt[0].text.strip().replace(u'\xa0', u''))
end_date = prompt[1].text
try:
actual_value = int(prompt[2].text[:-1].strip().replace(u'\xa0', u''))
aimed_value = int(sub_prompt[2].text[3:-1].strip().replace(u'\xa0', u''))
except ValueError:
actual_value, aimed_value = 'No monetary information', 'No monetary information'
return campaign.Campaign(link, title, desc, project_holder, actual_value, aimed_value, end_date, nb_contrib,
completion_rate, cats)
def test_tiny_campaign(self):
print("\n---- Tiny Test Start")
cycles = 60
goal = 40
refresh_rate = 1
max_demand = 3
c = campaign.Campaign(goal=goal,
cycles=cycles,
refresh_rate=refresh_rate,
max_demand=max_demand).run_campaign()
ideal_impression_counts_at_5_percent_increments = [
int((goal / 20) * x) for x in range(21)
]
for i in range(len(ideal_impression_counts_at_5_percent_increments)):
ideal_impression_count = ideal_impression_counts_at_5_percent_increments[
i]
actual_impression_count = c.checkpoint_impressions[i]
delta = max(max_demand, round(ideal_impression_count * 0.05))
print(
'Comparing ideal ({}) with actual ({}) to be within delta ({})'
.format(ideal_impression_count, actual_impression_count,
delta))
self.assertAlmostEqual(actual_impression_count,
ideal_impression_count,
delta=delta)
print("---- Tiny Test End")
def __init__(self, configfile=None):
'''
Initialization method of Choronzon. Reads the configuration,
instantiates objects of the vital classes, builds and
analyzes the first generation of chromosomes by reading
the initial population provided to the fuzzer.
'''
# configuration is a singleton
self.configuration = configuration.Configuration(configfile)
self.campaign = campaign.Campaign(self.configuration['CampaignName'])
seedpath = self.campaign.copy_directory(
self.configuration['InitialPopulation'], name='seedfiles')
self.tracer = tracer.Tracer()
self.strategy = strategy.FuzzingStrategy()
self.population = world.Population(self.tracer.cache)
self.evaluator = evaluator.Evaluator(self.tracer.cache)
try:
self.sharedpath = self.campaign.create_shared_directory(
self.configuration['ChromosomeShared'])
except:
self.sharedpath = None
# Initialize factory for building chromosome
# and the proxy for computing the fitness.
chromosomes = chromosome.Factory.build(seedpath)
for chromo in chromosomes:
self.population.add_chromosome(chromo)
self.analyze()
def test_campaign_generator( screen ):
camp = campaign.Campaign()
for t in range( 100 ):
nart = narrator.Narrative( camp, narrator.plots.PlotState(rank=t%5+1), adv_type="SHORTIE",start_rank=t%5+1, end_rank=5 )
#nart.build()
print t
for p in narrator.UNSORTED_PLOT_LIST:
if p._used > 0:
print "{} [{}]".format( p, p._used )
def __init__(self, epoch=0):
self.campaign = campaign.Campaign()
self.epoch = epoch
# This is a dictionary with key uid and value the chromosome object
self.chromosomes = dict()
self.selector = None
self.trace = tracer.Trace()
self.max_metrics = dict()
self.min_metrics = dict()
def start_campaign( init, screen ):
pygwrap.please_stand_by( screen, "Building world..." )
nart = narrator.Narrative( campaign.Campaign(), init )
if nart.story:
nart.build()
camp = nart.camp
pcs = choose_characters( screen )
if pcs:
camp.name = pygwrap.input_string(screen, redrawer=PosterRedraw(screen), prompt="Enter campaign name" )
camp.add_party( pcs )
camp.place_party()
camp.play( screen )
def initialize_campaign(self):
'''
Initiliaze a new tracer campaign.
'''
print '[+] Initializing campaign...'
self.campaign = campaign.Campaign(self.configuration['CampaignName'])
print '[+] Parsing whitelist...'
for target in self.configuration['Whitelist']:
exe = self.campaign.add(target)
print ' [-] Disassemblying %s...' % (os.path.basename(exe))
self.disassemble(exe)
def bardic_start_campaign( screen ):
init = narrator.plots.PlotState(rank=1)
pygwrap.please_stand_by( screen, "Building world..." )
nart = narrator.Narrative( campaign.Campaign(), init, adv_type="STUB_BARDIC", end_rank=5 )
if nart.story:
nart.build()
camp = nart.camp
pcs = choose_characters( screen )
if pcs:
camp.name = pygwrap.input_string(screen, redrawer=PosterRedraw(screen), prompt="Enter campaign name" )
camp.add_party( pcs )
camp.place_party()
camp.play( screen )
def __init__(self, cache, configfile=None):
self.cache = cache
self.configuration = configuration.Configuration(configfile)
self.campaign = campaign.Campaign()
self.load_metric_algorithms(self.configuration['FitnessAlgorithms'])
def main():
# set of testing matricies
matricies = []
# keep the selected parameters for reuse
print("===============================================================")
print("Campaign title and name ")
print("===============================================================")
campaignName = input("Name of campaign : ")
campaignUser = input("Your Name : ")
print("===============================================================")
print("Seed ")
print("===============================================================")
seedForce = None
sSeedForce = input("Force seed ? (None default): ")
if sSeedForce:
seedForce = int(sSeedForce)
# END IF
print("===============================================================")
print("Job Set size ")
print("===============================================================")
nN = int(input("Jobs number (begin) : "))
sNe = input("Jobs number (end) ("+str(nN)+" default) : ")
N_NumberBegin = nN
N_NumberEnd = nN
if sNe:
if int(sNe) > nN:
N_NumberEnd = int(sNe)
# END IF
# END IF
print("===============================================================")
print("Machines number ")
print("===============================================================")
nMmachineNumber = int(input("Machines number (begin) : "))
sMmachineNumbere = input("Machines number (end) ("+str(nMmachineNumber)+"default) : ")
M_NumberBegin = nMmachineNumber
M_NumberEnd = nMmachineNumber
if sMmachineNumbere:
if int(sMmachineNumbere) > nMmachineNumber:
M_NumberEnd = int(sMmachineNumbere)
# END IF
# END IF
print("===============================================================")
print("Job set generation methods ")
print("===============================================================")
matUniformNumber = int(input("How many uniform matricies to generate : "))
matNonUniformNumber = int(input("How many non uniform matricies to generate : "))
matGammaNumber = int(input("How many Gamma matricies to generate : "))
matBetaNumber = int(input("How many Beta matricies to generate : "))
matExponentialNumber= int(input("How many Exponential matricies to generate : "))
print("===============================================================")
print("From Parallel WorkLoad Archive ")
print("===============================================================")
matRealFiles = pwa.pwaFileChoice()
print("===============================================================")
print("Properties of generation ")
print("===============================================================")
nAb = 1.0
nBb = 100.0
nAlpah = 1.0
nBeta = 1.0
nLambda = 1.0
filename=""
if (matUniformNumber > 0 or matNonUniformNumber > 0):
nAb = float(input("a parameter : "))
nBb = float(input("b parameter : "))
if (matGammaNumber > 0 or matBetaNumber > 0):
nAlpah = float(input("alpha parameter (for gamma and beta) : "))
if (matGammaNumber > 0):
nBeta = float(input("beta parameter (for gamma) : "))
if (matExponentialNumber>0):
nLambda = float(input("lambda parameter (for ecxponential) : "))
print("===============================================================")
print("Algorithms ")
print("===============================================================")
useLPT = int(input("Use LPT rule ? : (1 yes, 0 no) : "))
useSLACK = int(input("UParamFilese Slack ? : (1 yes, 0 no) : "))
useLDM = int(input("Use LDM ? : (1 yes, 0 no) : "))
useCOMBINE = int(input("Use COMBINE ? : (1 yes, 0 no) : "))
useMULTIFIT = int(input("Use MULTIFIT ? : (1 yes, 0 no) : "))
#======================================================================
# Parameters file (json)
#======================================================================
if s.EXP_PARAMETERS:
fileSetup = s.ParamFile()
fileSetup.create(campaignName,campaignUser,
seedForce,N_NumberBegin,N_NumberEnd,M_NumberBegin,M_NumberEnd,
matUniformNumber,matNonUniformNumber,matGammaNumber,matBetaNumber,matExponentialNumber,
matRealFiles,
nAb, nBb,nAlpah,nBeta,nLambda,
useLPT,useSLACK,useLDM,useCOMBINE,useMULTIFIT)
# END IF
print("===============================================================")
print("Results computation ")
print("===============================================================")
c = cp.Campaign(campaignName, campaignUser, N_NumberBegin, N_NumberEnd, M_NumberBegin, M_NumberEnd, matUniformNumber, matNonUniformNumber, matGammaNumber, matBetaNumber, matExponentialNumber, matRealFiles, nAb, nBb, nAlpah, nBeta, nLambda, seedForce)
#
if useLPT==1: c.runAlgorithm(cmm.lpt)
if useSLACK==1: c.runAlgorithm(cmm.slack)
if useLDM==1: c.runAlgorithm(cmm.ldm)
if useCOMBINE==1: c.runAlgorithm(cmm.combine)
if useMULTIFIT==1: c.runAlgorithm(cmm.multifit)
#
c.exportCSV()
def main():
# #######################################################################
# #
# PARAMETERS TO BE MODIFIED #
# #
# Modify the desired values #
# save py file (ctrl+s) #
# then press F5 to execute (if idle is used) #
# #
# #######################################################################
#Campaign title and name ==============================================
campNames_5 = [
"dCroce_1_100_i50_m5_n10_50_100_500_1000_",
"dCroce_1_1000_i50_m5_n10_50_100_500_1000_",
"dCroce_1_10000_i50_m5_n10_50_100_500_1000_"
]
campNames_x = [
"dCroce_1_100_i40_m10_25_n50_100_500_1000_",
"dCroce_1_1000_i40_m10_25_n50_100_500_1000_",
"dCroce_1_10000_i40_m10_25_n50_100_500_1000_"
]
bValues = [100, 1000, 10000]
campaignUser = "******" # string
# Seed ================================================================
seedForce = None # integer
# Job Set size with N_NumberEnd >= N_NumberBegin ======================
N_NumberBegin = 0 # int
N_NumberEnd = 0 # int
# Machines number with M_NumberEnd >= M_NumberBegin ===================
M_NumberBegin = 1 # int
M_NumberEnd = 0 # int
# Job set generation methods ===========================================
matUniformNumber = 0 # int
matNonUniformNumber = 0 # int
matGammaNumber = 10 # int
matBetaNumber = 0 # int
matExponentialNumber = 0 # int
# From Parallel WorkLoad Archive ======================================
# pwa.pwaFileChoice(X) if None asc you witch file to use eg pwa.pwaFileChoice()
# if 1 use all files present in the PWA folder
# if 0 does not use any of the files present
matRealFiles = pwa.pwaFileChoice(0)
# Properties of generation ============================================
nAlpah = 1.0 # float for gamma (k) and beta (alpha)
nBeta = 1.0 # float for gamma (theta)
nLambda = 1.0 # float for exponential (lambda)
#Algorithms ==========================================================
useLPT = 1 # 1 or 0
useSLACK = 1 # 1 or 0
useLDM = 1 # 1 or 0
useCOMBINE = 1 # 1 or 0
useMULTIFIT = 0 # 1 or 0
# #######################################################################
# #
# APPLICATION PART #
# do not change anything here #
# #
# #######################################################################
print("===============================================================")
print("Results computation ")
print("===============================================================")
for camp in range(10):
for b in range(3):
#====================================================================
# 1 - 100x
#====================================================================
nAb = 1.0
nBb = bValues[b]
#--------------------------------------------------------------------
# i50 with 5 machines
#--------------------------------------------------------------------
campaignName = campNames_5[b] + str(camp)
# LIST OF JOBS Numbers
N_List = [10, 50, 100, 500, 1000]
# LIST OF MACHINES Number
M_List = [5]
# Parameters file (json)
if s.EXP_PARAMETERS:
fileSetup = s.ParamFile()
fileSetup.create(campaignName, campaignUser, seedForce,
N_NumberBegin, N_NumberEnd, N_List,
M_NumberBegin, M_NumberEnd, M_List,
matUniformNumber, matNonUniformNumber,
matGammaNumber, matBetaNumber,
matExponentialNumber, matRealFiles, nAb, nBb,
nAlpah, nBeta, nLambda, useLPT, useSLACK,
useLDM, useCOMBINE, useMULTIFIT)
# Create campaign
c = cp.Campaign(campaignName, campaignUser, N_NumberBegin,
N_NumberEnd, N_List, M_NumberBegin, M_NumberEnd,
M_List, matUniformNumber, matNonUniformNumber,
matGammaNumber, matBetaNumber,
matExponentialNumber, matRealFiles, nAb, nBb,
nAlpah, nBeta, nLambda, seedForce)
# Apply algorithms
if useLPT == 1: c.runAlgorithm(cmm.lpt)
if useSLACK == 1: c.runAlgorithm(cmm.slack)
if useLDM == 1: c.runAlgorithm(cmm.ldm)
if useCOMBINE == 1: c.runAlgorithm(cmm.combine)
if useMULTIFIT == 1: c.runAlgorithm(cmm.multifit)
c.exportCSV()
#--------------------------------------------------------------------
# i40 with 10, 25 machines
#--------------------------------------------------------------------
campaignName = campNames_x[b] + str(camp)
# LIST OF JOBS Numbers
N_List = [50, 100, 500, 1000]
# LIST OF MACHINES Number
M_List = [10, 25]
# Parameters file (json)
if s.EXP_PARAMETERS:
fileSetup = s.ParamFile()
fileSetup.create(campaignName, campaignUser, seedForce,
N_NumberBegin, N_NumberEnd, N_List,
M_NumberBegin, M_NumberEnd, M_List,
matUniformNumber, matNonUniformNumber,
matGammaNumber, matBetaNumber,
matExponentialNumber, matRealFiles, nAb, nBb,
nAlpah, nBeta, nLambda, useLPT, useSLACK,
useLDM, useCOMBINE, useMULTIFIT)
# Create campaign
c = cp.Campaign(campaignName, campaignUser, N_NumberBegin,
N_NumberEnd, N_List, M_NumberBegin, M_NumberEnd,
M_List, matUniformNumber, matNonUniformNumber,
matGammaNumber, matBetaNumber,
matExponentialNumber, matRealFiles, nAb, nBb,
nAlpah, nBeta, nLambda, seedForce)
# Apply algorithms
if useLPT == 1: c.runAlgorithm(cmm.lpt)
if useSLACK == 1: c.runAlgorithm(cmm.slack)
if useLDM == 1: c.runAlgorithm(cmm.ldm)
if useCOMBINE == 1: c.runAlgorithm(cmm.combine)
if useMULTIFIT == 1: c.runAlgorithm(cmm.multifit)
c.exportCSV()
def main():
# #######################################################################
# #
# #######################################################################
root = tk.Tk()
root.withdraw() # pour ne pas afficher la fenêtre Tk
name = askopenfilename(initialdir=s.folder(s.FOLDER_RESULTS),
filetypes=[("json parameters files", "*.json")])
if not name: return
param = s.ParamFile()
d = param.read(name) # D is a dictionnary with keys parameters
#Campaign title and name ==============================================
campaignName = d["campaignName"] + "_reloaded"
campaignUser = d["campaignUser"]
# Seed ================================================================
seedForce = d["seedForce"]
# Job Set size with N_NumberEnd >= N_NumberBegin ======================
N_NumberBegin = d["N_NumberBegin"]
N_NumberEnd = d["N_NumberEnd"]
N_List = d["N_List"]
# Machines number with M_NumberEnd >= M_NumberBegin ===================
M_NumberBegin = d["M_NumberBegin"]
M_NumberEnd = d["M_NumberEnd"]
M_List = d["M_List"]
# Job set generation methods ===========================================
matUniformNumber = d["matUniformNumber"]
matNonUniformNumber = d["matNonUniformNumber"]
matGammaNumber = d["matGammaNumber"]
matBetaNumber = d["matBetaNumber"]
matExponentialNumber = d["matExponentialNumber"]
# From Parallel WorkLoad Archive ======================================
# pwa.pwaFileChoice(X) if None asc you witch file to use eg pwa.pwaFileChoice()
# if 1 use all files present in the PWA folder
# if 0 does not use any of the files present
matRealFiles = d["matRealFiles"]
# Properties of generation ============================================
nAb = d["A"]
nBb = d["B"]
nAlpah = d["Alpha"]
nBeta = d["Beta"]
nLambda = d["Lambda"]
#Algorithms ==========================================================
useLPT = d["useLPT"]
useSLACK = d["useSLACK"]
useLDM = d["useLDM"]
useCOMBINE = d["useCOMBINE"]
useMULTIFIT = d["useMULTIFIT"]
# #######################################################################
# #
# APPLICATION PART #
# do not change anything here #
# #
# #######################################################################
#======================================================================
# Parameters file (json)
#======================================================================
if s.EXP_PARAMETERS:
fileSetup = s.ParamFile()
fileSetup.create(campaignName, campaignUser, seedForce, N_NumberBegin,
N_NumberEnd, N_List, M_NumberBegin, M_NumberEnd,
M_List, matUniformNumber, matNonUniformNumber,
matGammaNumber, matBetaNumber, matExponentialNumber,
matRealFiles, nAb, nBb, nAlpah, nBeta, nLambda,
useLPT, useSLACK, useLDM, useCOMBINE, useMULTIFIT)
# END IF
print("===============================================================")
print("Results computation ")
print("===============================================================")
c = cp.Campaign(campaignName, campaignUser, N_NumberBegin, N_NumberEnd,
N_List, M_NumberBegin, M_NumberEnd, M_List,
matUniformNumber, matNonUniformNumber, matGammaNumber,
matBetaNumber, matExponentialNumber, matRealFiles, nAb,
nBb, nAlpah, nBeta, nLambda, seedForce)
#
if useLPT == 1: c.runAlgorithm(cmm.lpt)
if useSLACK == 1: c.runAlgorithm(cmm.slack)
if useLDM == 1: c.runAlgorithm(cmm.ldm)
if useCOMBINE == 1: c.runAlgorithm(cmm.combine)
if useMULTIFIT == 1: c.runAlgorithm(cmm.multifit)
#
c.exportCSV()
def __init__(self, seeddir):
self.configuration = configuration.Configuration()
self.campaign = campaign.Campaign()
if seeddir != None:
self.directory = self._check_path(seeddir)
self._load_parser()
def main():
# #######################################################################
# #
# PARAMETERS TO BE MODIFIED #
# #
# Modify the desired values #
# save py file (ctrl+s) #
# then press F5 to execute (if idle is used) #
# #
# #######################################################################
#Campaign title and name ==============================================
campaignName = "uni_test3d_m5_50n10_1000i_gamma_LDM" # string
campaignUser = "******" # string
# Seed ================================================================
seedForce = None # integer
# Job Set size with N_NumberEnd >= N_NumberBegin ======================
N_NumberBegin = 10 # int
N_NumberEnd = 1000 # int
# OR LIST OF JOBS Numbers (N_List has priority. use N_NumberBegin and N_NumberEnd if N_List = [])
N_List = []
# Machines number with M_NumberEnd >= M_NumberBegin ===================
M_NumberBegin = 5 # int
M_NumberEnd = 50 # int
# OR LIST OF JOBS Numbers (N_List has priority. use M_NumberBegin and M_NumberEnd if M_List = [])
M_List = []
# Job set generation methods ===========================================
matUniformNumber = 0 # int
matNonUniformNumber = 0 # int
matGammaNumber = 1 # int
matBetaNumber = 0 # int
matExponentialNumber= 0 # int
# From Parallel WorkLoad Archive ======================================
# pwa.pwaFileChoice(X) if None asc you witch file to use eg pwa.pwaFileChoice()
# if 1 use all files present in the PWA folder
# if 0 does not use any of the files present
matRealFiles = pwa.pwaFileChoice(0)
# Properties of generation ============================================
nAb = 1.0 # float
nBb = 1000.0 # float
nAlpah = 1.0 # float for gamma (k) and beta (alpha)
nBeta = 1.0 # float for gamma (theta)
nLambda = 1.0 # float for exponential (lambda)
#Algorithms ==========================================================
useLPT = 0 # 1 or 0
useSLACK = 0 # 1 or 0
useLDM = 1 # 1 or 0
useCOMBINE = 0 # 1 or 0
useMULTIFIT = 0 # 1 or 0
# #######################################################################
# #
# APPLICATION PART #
# do not change anything here #
# #
# #######################################################################
#======================================================================
# Parameters file (json)
#======================================================================
if s.EXP_PARAMETERS:
fileSetup = s.ParamFile()
fileSetup.create(campaignName,campaignUser,
seedForce,N_NumberBegin,N_NumberEnd,N_List, M_NumberBegin,M_NumberEnd, M_List,
matUniformNumber,matNonUniformNumber,matGammaNumber,matBetaNumber,matExponentialNumber,
matRealFiles,
nAb, nBb,nAlpah,nBeta,nLambda,
useLPT,useSLACK,useLDM,useCOMBINE,useMULTIFIT)
print("===============================================================")
print("Results computation ")
print("===============================================================")
c = cp.Campaign(campaignName, campaignUser,
N_NumberBegin, N_NumberEnd, N_List,
M_NumberBegin, M_NumberEnd, M_List,
matUniformNumber,
matNonUniformNumber,
matGammaNumber,
matBetaNumber,
matExponentialNumber,
matRealFiles,
nAb,
nBb,
nAlpah,
nBeta,
nLambda,
seedForce)
#
if useLPT==1: c.runAlgorithm(cmm.lpt)
if useSLACK==1: c.runAlgorithm(cmm.slack)
if useLDM==1: c.runAlgorithm(cmm.ldm)
if useCOMBINE==1: c.runAlgorithm(cmm.combine)
if useMULTIFIT==1: c.runAlgorithm(cmm.multifit)
c.exportCSV()
