def addDomainsForVhost(self, vhostID): vhostField = self.env.db.querySingle("SELECT optionID \ FROM cp" + self.env.cpnr + "_domain_option \ WHERE optionName = 'vhostContainerID'" ) domains = self.env.db.query("SELECT domainID \ FROM cp" + self.env.cpnr + "_domain_option_value \ WHERE domainOption" + str(vhostField[0]) + " = " + str(vhostID)) for d in domains: if self.domainIDs.count(d[0]) == 0: self.addDomainToVhost(domain(d[0], self.env)) self.domainIDs.append(d[0]) # get all subdomains for this domain subdomains = self.env.db.query("SELECT domainID \ FROM cp" + self.env.cpnr + "_domain \ WHERE parentDomainID = " + str(d[0])) for s in subdomains: if self.domainIDs.count(s[0]) == 0: self.addDomainToVhost(domain(s[0], self.env)) self.domainIDs.append(s[0])
def __init__(self, gravity=True, units="MKS"): self.mesh = mesh.mesh() self.domain = domain.domain() self.forces = [] self.constraints = [] self.globalStiffnessMatrix = np.array([]) self.globalForceVector = [] self.globalDisplacementVector = []
def main(): today = datetime.datetime.today() one_day = datetime.timedelta(1) yesterday = today - one_day # find yesterday's log logfile = "/var/log/gogogon/consumer.log.%04d-%02d-%02d" % \ (yesterday.year, yesterday.month, yesterday.day) if not os.path.exists(logfile): return # sort and uniq the log cmd = 'grep INFO %s | cut -f 4- -d " " | sort | uniq -c' % logfile pipe = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) # collect up global hashes and click counts details = dict() for line in pipe.stdout: (count, global_hash, url) = line.strip().split(' ', 2) details[global_hash] = dict( u=url, global_clicks=count, agency=domain(url), global_hash=global_hash, ) # grab hashes in groups of GROUPSIZE size for i in xrange(1+len(details)/GROUPSIZE): hashes = details.keys()[i*GROUPSIZE:i*GROUPSIZE+GROUPSIZE] # lookup titles for info in bitly.info(*hashes): if not info['title']: continue details[info['hash']]['title']=info['title'] # output files json_file = "/var/log/gogogon/ranks/%04d-%02d-%02d.json" % \ (yesterday.year, yesterday.month, yesterday.day) csv_file = "/var/log/gogogon/ranks/%04d-%02d-%02d.csv" % \ (yesterday.year, yesterday.month, yesterday.day) # sort by global clicks descending records = details.values() records.sort(key=lambda x: x["global_clicks"], reverse=True) # write json json.dump(records, file(json_file, 'w')) # write csv csv_writer = csv.writer(file(csv_file, 'w')) csv_writer.writerow(["Long URL", "Page Title", "Clicks", "Agency Domain", "Global hash"]) for record in records: if not 'title' in record: continue csv_writer.writerow([ record['u'], record['title'].encode('utf8'), record['global_clicks'], record['agency'], record['global_hash'], ])
def addDomainsForUser(self, userID): domains = self.env.db.query("SELECT domainID \ FROM cp" + self.env.cpnr + "_domain \ WHERE userID = " + userID) for d in domains: if self.domainIDs.count(domainID) == 0: self.addDomainToVhost(domain(d[0], self.env)) self.domainIDs.append(d[0])
def setUp(self): domain = dm.domain(np.array([0, 0]), np.array([1, 1])) inDomain = lambda x, y: 0 < x < 1 and 0 < y < 1 onBoundary = lambda x, y: abs(x) < np.spacing(1) or abs(x-1) < np.spacing(1) \ or abs(y-1) < np.spacing(1) or abs(y) < np.spacing(1) getBoundaryValue = lambda x, y: 0 diri = dr.dirichlet_bc(inDomain, onBoundary, getBoundaryValue, domain) self.solver = poisson.poisson_solver( lambda x, y: 32 * x * (1 - x) + 32 * y * (1 - y), diri)
def setUp(self): domain = dm.domain(np.array([-1, -1]), np.array([1, 1])) inDomain = lambda x, y: x**2 + y**2 < 1 onBoundary = lambda x, y: abs(x**2 + y**2 - 1) < np.spacing(1) getBoundaryValue = lambda x, y: 1 getNearestPoint = [get_x_by_y, get_y_by_x] diri = dr.dirichlet_bc(inDomain, onBoundary, getBoundaryValue, domain, getNearestPoint) self.solver = fdm.fdm_solver(None, lambda x, y: 16 * (x**2 + y**2), diri)
def setUp(self): domain = dm.domain(np.array([-1, -1]), np.array([1, 1])) d = ddx.ddx(0.1) self.expression = expr.diff_operator_expression([d]) inDomain = lambda x, y: abs(x) < 1 and abs(y) < 1 onBoundary = lambda x, y: abs(x-1) < np.spacing(1) or abs(x+1) < np.spacing(1) \ or abs(y-1) < np.spacing(1) or abs(y+1) < np.spacing(1) getBoundaryValue = lambda x, y: 1 self.diri = dr.dirichlet_bc(inDomain, onBoundary, getBoundaryValue, domain) self.diri2 = dr.dirichlet_bc(inDomain, onBoundary, getBoundaryValue, domain, (lambda x, y: 1, lambda x, y: 2)) self.solver = fdm.fdm_solver(self.expression, lambda x, y: 1, self.diri) self.solver2 = fdm.fdm_solver(self.expression, lambda x, y: 1, self.diri2) domain_s = dm.domain(np.array([0, 0]), np.array([1, 1])) inDomain_s = lambda x, y: 0 < x < 1 and 0 < y < 1 onBoundary_s = lambda x, y: abs(x-1) < np.spacing(1) or abs(x) < np.spacing(1) \ or abs(y-1) < np.spacing(1) or abs(y) < np.spacing(1) getBoundaryValue_s = lambda x, y: 1/(1+x) + 1/(1+y) f_s = lambda x, y: 2/(1+x)**3 + 2/(1+y)**3 self.dirichlet = dr.dirichlet_bc(inDomain_s, onBoundary_s, getBoundaryValue_s, domain_s) self.real_solver = fdm.fdm_solver([], f_s, self.dirichlet)
def __init__(self, node): self.type = "drs" assert node.tag == "drs" self.expression = [] for subnode in node: if subnode.tag == "domain": self.expression.append(domain(subnode)) elif subnode.tag == "conds": self.expression.append(conds(subnode)) else: print subnode.tag assert False, "unrecognized node"
def addDomainsForUser(self, userID): domains = self.env.db.query( "SELECT domainID \ FROM cp" + self.env.cpnr + "_domain \ WHERE userID = " + userID ) for d in domains: if self.domainIDs.count(domainID) == 0: self.addDomainToVhost(domain(d[0], self.env)) self.domainIDs.append(d[0])
def addDomainsForVhost(self, vhostID): vhostField = self.env.db.querySingle( "SELECT optionID \ FROM cp" + self.env.cpnr + "_domain_option \ WHERE optionName = 'vhostContainerID'" ) domains = self.env.db.query( "SELECT domainID \ FROM cp" + self.env.cpnr + "_domain_option_value \ WHERE domainOption" + str(vhostField[0]) + " = " + str(vhostID) ) for d in domains: if self.domainIDs.count(d[0]) == 0: self.addDomainToVhost(domain(d[0], self.env)) self.domainIDs.append(d[0]) # get all subdomains for this domain subdomains = self.env.db.query( "SELECT domainID \ FROM cp" + self.env.cpnr + "_domain \ WHERE parentDomainID = " + str(d[0]) ) for s in subdomains: if self.domainIDs.count(s[0]) == 0: self.addDomainToVhost(domain(s[0], self.env)) self.domainIDs.append(s[0])
def setUp(self): td_domain = dm.domain(np.array([0, 0]), np.array([1, 1])) td_inDomain = lambda x, y: 0 < x < 1 and 0 < y < 1 td_onBoundary = lambda x, y: abs(x-1) < np.spacing(1) or abs(x) < np.spacing(1) \ or abs(y) < np.spacing(1) def td_getBV(x, y): if abs(x) < np.spacing(1) or abs(x-1) < np.spacing(1): return 0 else: #return math.sin(math.pi*x) #return 6*math.sin(math.pi*x) + 3*math.cos(2*math.pi*x) - 3 return 6*x * math.sin(5*math.pi*x) f_s = lambda x, y: 0 dirichlet = dr.dirichlet_bc(td_inDomain, td_onBoundary, td_getBV, td_domain) self.solver = fdm.fdm_solver([], f_s, dirichlet)
def setUp(self): td_domain = dm.domain(np.array([0, 0]), np.array([1, 1])) td_inDomain = lambda x, y: 0 < x < 1 and 0 < y < 1 td_onBoundary = lambda x, y: abs(x-1) < np.spacing(1) or abs(x) < np.spacing(1) \ or abs(y) < np.spacing(1) or abs(y-1) < np.spacing(1) def td_getBV(x, y): if abs(x) < np.spacing(1) or abs(x - 1) < np.spacing(1) or abs( y) < np.spacing(1) or abs(y - 1) < np.spacing(1): return 0 elif y <= 1 / 2 and (y <= x and y <= -x + 1): return 20 * y elif x <= 1 / 2 and (y >= x and y <= -x + 1): return 20 * x elif x >= 1 / 2 and (y <= x and y >= -x + 1): return 20 - 20 * x else: return 20 - 20 * y f_s = lambda x, y: 0 dirichlet = dr.dirichlet_rectangular_bc(td_inDomain, td_onBoundary, td_getBV, td_domain, 2) self.solver = fdm.fdm_solver([], f_s, dirichlet)
def main(): ymd = sys.argv[1] print "recovering %s" % ymd # gather global hashes and click counts details = dict() lines = os.popen("curl -s %s | grep %s" % (ARCHIVE_URL, ymd)).readlines() for line in lines: matches = LINK_RE.findall(line) if matches: link = ARCHIVE_URL+matches[0] for (global_hash, url) in read_data(link): if global_hash not in details: details[global_hash] = dict( u=url, global_clicks=1, agency=domain(url), global_hash=global_hash, ) else: details[global_hash]['global_clicks'] += 1 print "getting titles" # grab hashes in groups of GROUPSIZE size for i in xrange(1+len(details)/GROUPSIZE): hashes = details.keys()[i*GROUPSIZE:i*GROUPSIZE+GROUPSIZE] # lookup titles for item in bitly.info(hashes=hashes): if 'title' not in item: continue details[item['hash']]['title']=item['title'] # sort by global clicks descending records = details.values() records.sort(key=lambda x: x["global_clicks"], reverse=True) write_output_files(records=records, ymd=ymd, latest=False)
def addDomain(self, domainID): if self.domainIDs.count(domainID) == 0: self.addDomainToVhost(domain(domainID, self.env)) self.domainIDs.append(domainID)
def main(): # setup logger formatter = logging.Formatter( '%(process)d %(levelname)s %(asctime)s %(message)s', '%Y-%m-%d %H:%M:%S') handler = logging.FileHandler("/var/log/gogogon/ranks.log") handler.setFormatter(formatter) logger = logging.getLogger() logger.addHandler(handler) logger.setLevel(logging.DEBUG) logger.debug("starting up") today = datetime.datetime.today() one_day = datetime.timedelta(1) yesterday = today - one_day ymd = "%04d-%02d-%02d" % (yesterday.year, yesterday.month, yesterday.day) # find yesterday's log logfile = os.path.join(LOG_INPUT_DIR, "consumer.log.%s" % ymd) # But allow this to be overridden parser = optparse.OptionParser() parser.add_option('-f', '--file', dest="logfile", default=logfile) parser.add_option('-o', '--output-directory', dest="output_dir", default=RANKS_OUTPUT_DIR) parser.add_option('-a', '--agency', dest="use_agency_domain", default=False) options, remainder = parser.parse_args() logfile = options.logfile output_dir = options.output_dir if not os.path.exists(logfile): raise RuntimeError('Log file does not exist: ' + logfile) if not os.path.exists(output_dir): raise RuntimeError('Output directory does not exist: ' + output_dir) # sort and uniq the log cmd = 'grep INFO %s | cut -f 5- -d " " | sort | uniq -c' % logfile pipe = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) # collect up global hashes and click counts details = dict() for line in pipe.stdout: (count, global_hash, url) = line.strip().split(' ', 2) details[global_hash] = dict( u=url, global_clicks=long(count), agency=domain(url), global_hash=global_hash, ) # grab hashes in groups of GROUPSIZE size for i in xrange(1 + len(details) / GROUPSIZE): hashes = details.keys()[i * GROUPSIZE:i * GROUPSIZE + GROUPSIZE] # lookup titles for item in bitly.info(hashes=hashes): if 'title' not in item: continue details[item['hash']]['title'] = item['title'] # lookup yesterday's clicks for item in bitly.clicks_by_day(hashes=hashes, days=2): if 'clicks' not in item: continue clicks = int(item['clicks'][1]['clicks']) if clicks > details[item['hash']]['global_clicks']: details[item['hash']]['global_clicks'] = clicks # sort by global clicks descending records = details.values() records.sort(key=lambda x: x["global_clicks"], reverse=True) logger.debug("writing output files") write_output_files(records, ymd, output_dir) if options.use_agency_domain: write_agency_domain_files(records, output_dir, ymd) logger.debug("shutting down")
neutralWebRep, poorWebRep, goodWebRep, unknownWebRep = 0, 0, 0, 0 numOfHASHES = len(lst_hash) undetectedHASHES = 0 document = Document() document.add_heading('Reporte Automatización de IoCs', 0) d = document.add_paragraph("", style="List Bullet") d.add_run("DETALLE").bold = True urlText = document.add_paragraph("", style="List Bullet 2") urlText.add_run("URLs").bold = True urlCounter = 1 for url in lst_url: objUrlDomain = dm.domain(url) print(objUrlDomain.urlToDict()) document.add_paragraph("\t" + str(urlCounter) + ". " + objUrlDomain.url) geo = document.add_paragraph('\t\ta. Geolocalización: ') geo.add_run(objUrlDomain.location) ip = document.add_paragraph('\t\tb. IP: ') ip.add_run( objUrlDomain.ip if objUrlDomain.ip != "Unknown" else "Desconocida") repu = document.add_paragraph('\t\tc. Reputación: ') if objUrlDomain.reputation[1] == 'Poor': repu.add_run("Mala") poorURLS += 1 elif objUrlDomain.reputation[1] == 'Neutral': repu.add_run("Neutral") neutralURLS += 1 elif objUrlDomain.reputation[1] == 'Good':
def Create_config_on_lattice(latt: lattice, Directions, shuffle_mols=False, NSwaps=0): """ Create a configuration within a Mol_System object on a lattice latt. The lattice size can't be of size smaller than the requested number of species to be placed on the lattice. The Directions argument is a list of tuples. Each tuple must contain in element 0 the molecule/atom that will be placed in the lattice and in element 1 the number of species that the user wishes to place. Two OPTIONAL argument can be provided. If 'shuffle_mols' is True then when all the species have been inserted, we perform random position swaps between them. The default number of NShuffles is MolSys.NAtoms, but it can be changes through the last optional argument. """ NSpecies = [] NAtomsTot = 0 NBondsTot = 0 NAnglesTot = 0 # to begin we need to count the total number of atoms requested for Dir in Directions: Spc = deepcopy(Dir[0]) NSpeciesOfThis = int(Dir[1]) if type(Spc) == atom: NAtSpc = 1 NBonds = 0 NAngles = 0 elif type(Spc) == molecule: NAtSpc = Spc.NAtoms NBonds = Spc.NBonds NAngles = Spc.NAngles else: sys.exit( "Create_config_on_lattice : ERROR - Invalid species of type %s" % str(type(Spc))) NAtomsTot += NAtSpc * NSpeciesOfThis NBondsTot += NBonds * NSpeciesOfThis NAnglesTot += NAngles * NSpeciesOfThis NSpecies.append(NSpeciesOfThis) if np.sum(NSpecies) > latt.NNodes: sys.exit( "Create_config_on_lattice : ERROR - the specified lattice is not big enough" ) SimBox = domain(latt.edge) MolSys = Mol_System(SimBox) iter_latt = iter(latt) for Dir in Directions: Spc = deepcopy(Dir[0]) NSpeciesOfThis = int(Dir[1]) for iSpc in range(0, NSpeciesOfThis): xyz = next( iter_latt ) #+ 1.e-4 # add a small perturbation to avoid atoms on the surface of the lattice Spc.change_coords(xyz, SimBox) MolSys.insert_spc(Spc) MolSys.NSpeciesOf = np.copy(NSpecies) if (NAtomsTot != MolSys.NAtoms): print(NAtomsTot, MolSys.NAtoms) sys.exit( "Create_config_on_lattice : ERROR - miscounted the number of atoms" ) if (shuffle_mols): MolSys.rand_swaps(NSwaps) return MolSys
class Mol_System: NAtoms = 0 NBonds = 0 NAngles = 0 NAtomTypes = 0 NBondTypes = 0 NAngleTypes = 0 NSpcTypes = 0 NSpcOf = [] NSpc = 0 At = [] Bnd = [] Ang = [] Spc = [] Box = domain(np.zeros(3, np.double)) def __init__(self, Box: domain): self.Box = deepcopy(Box) pass def insert_spc(self, Spc): """ This function should be used when inserting molecules or atoms in a Mol_System. An Spc can be either an atom or a molecule. """ SpcIn = deepcopy(Spc) iSpc = self.NSpc if type(SpcIn) == molecule: SpcIn.iSpc = iSpc iSpc = self.__insert_mol(SpcIn) elif type(SpcIn) == atom: SpcIn.iSpc = iSpc idx = self.__insert_atom(SpcIn) else: sys.exit("Mol_System.insert_spc : ERROR - Unrecognised species") self.Spc.append(SpcIn) self.NSpc += 1 def rand_swaps(self, NSwaps, TargetSlice=slice(0, -1, 1)): for iSwap in range(NSwaps): Spc1 = random.choice(self.Spc) Spc2 = random.choice(self.Spc[TargetSlice]) xyz1 = Spc1.center() xyz2 = Spc2.center() Spc1.change_coords(xyz2, self.Box) Spc2.change_coords(xyz1, self.Box) def __insert_atom(self, atomIn: atom): if (atomIn.iSpc > self.NSpcTypes): self.__expand_SpcArrays(atomIn.iSpc) if (atomIn.Type + 1 > self.NAtomTypes): self.NAtomTypes += atomIn.Type - self.NAtomTypes + 1 atomIn.idx = self.NAtoms self.At.append(atomIn) self.NAtoms += 1 return atomIn.idx def __insert_bond(self, BndIn: bond): if (BndIn.Type + 1 > self.NBondTypes): self.NBondTypes += BndIn.Type - self.NBondTypes + 1 BndIn.idx = self.NBonds self.Bnd.append(BndIn) self.NBonds += 1 return BndIn.idx def __insert_angle(self, AngIn: angle): if (AngIn.Type + 1 > self.NAngleTypes): self.NAngleTypes += AngIn.Type - self.NAngleTypes + 1 AngIn.idx = self.NAngles self.Ang.append(AngIn) self.NAngles += 1 return AngIn.idx def __insert_mol(self, mol: molecule): for AtomIn in mol.At: AtomIn.iSpc = mol.iSpc old_idx = AtomIn.idx new_idx = self.__insert_atom(AtomIn) idx_shift = new_idx - old_idx for BndIn in mol.Bnd: BndIn.At += idx_shift idx = self.__insert_bond(BndIn) for AngIn in mol.Ang: AngIn.At += idx_shift idx = self.__insert_angle(AngIn) return mol.iSpc def __expand_SpcArrays(self, MaxTypeIdx): HowManyMore = MaxTypeIdx - self.NSpcTypes + 1 padding = [0] * HowManyMore self.NSpcOf = self.NSpcOf + padding self.NSpcTypes += HowManyMore
def main(): # setup logger formatter = logging.Formatter('%(process)d %(levelname)s %(asctime)s %(message)s', '%Y-%m-%d %H:%M:%S') handler = logging.FileHandler("/var/log/gogogon/ranks.log") handler.setFormatter(formatter) logger = logging.getLogger() logger.addHandler(handler) logger.setLevel(logging.DEBUG) logger.debug("starting up") today = datetime.datetime.today() one_day = datetime.timedelta(1) yesterday = today - one_day ymd = "%04d-%02d-%02d" % (yesterday.year, yesterday.month, yesterday.day) # find yesterday's log logfile = os.path.join(LOG_INPUT_DIR, "consumer.log.%s" % ymd) # But allow this to be overridden parser = optparse.OptionParser() parser.add_option('-f', '--file', dest="logfile", default=logfile) parser.add_option('-o', '--output-directory', dest="output_dir", default= RANKS_OUTPUT_DIR) parser.add_option('-a', '--agency', dest="use_agency_domain", default=False) options, remainder = parser.parse_args() logfile = options.logfile output_dir = options.output_dir if not os.path.exists(logfile): raise RuntimeError('Log file does not exist: ' + logfile) if not os.path.exists(output_dir): raise RuntimeError('Output directory does not exist: ' + output_dir) # sort and uniq the log cmd = 'grep INFO %s | cut -f 5- -d " " | sort | uniq -c' % logfile pipe = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) # collect up global hashes and click counts details = dict() for line in pipe.stdout: (count, global_hash, url) = line.strip().split(' ', 2) details[global_hash] = dict( u=url, global_clicks=long(count), agency=domain(url), global_hash=global_hash, ) # grab hashes in groups of GROUPSIZE size for i in xrange(1+len(details)/GROUPSIZE): hashes = details.keys()[i*GROUPSIZE:i*GROUPSIZE+GROUPSIZE] # lookup titles for item in bitly.info(hashes=hashes): if 'title' not in item: continue details[item['hash']]['title']=item['title'] # lookup yesterday's clicks for item in bitly.clicks_by_day(hashes=hashes, days=2): if 'clicks' not in item: continue clicks = int(item['clicks'][1]['clicks']) if clicks > details[item['hash']]['global_clicks']: details[item['hash']]['global_clicks'] = clicks # sort by global clicks descending records = details.values() records.sort(key=lambda x: x["global_clicks"], reverse=True) logger.debug("writing output files") write_output_files(records, ymd, output_dir) if options.use_agency_domain: write_agency_domain_files(records, output_dir, ymd) logger.debug("shutting down")
def setUp(self): self.inDomain = lambda x, y: abs(x) < 1 and abs(y) < 1 self.onBoundary = lambda x, y: abs(x-1) < np.spacing(1) or abs(x+1) < np.spacing(1) \ or abs(y-1) < np.spacing(1) or abs(y+1) < np.spacing(1) self.getBoundaryValue = lambda x, y: 1 self.domain = dm.domain(np.array([-1, -1]), np.array([1, 1]))
import os import socket from domain import domain from config import config def get_ip(): sock = socket.create_connection(('ns1.dnspod.net', 6666)) ip = sock.recv(16) sock.close() return ip if __name__ == '__main__': api_key = config['api_key'] api_secret = config['api_secret'] domain_name = config['domain'] sub_domain = config['sub_domain'] d = domain(api_key, api_secret) resolve_list = d.resolve_list(domain_name, 1, 100) sub_domain_info = d.get_info_by_subdomain(resolve_list.result, sub_domain) ip = get_ip() ip = ip.decode('utf-8') if ip == sub_domain_info.rdata: os._exit(0) edit_info = d.resolve_edit(sub_domain_info.record_id, sub_domain, 'DEFAULT', 'A', 600, ip, domain_name) print(edit_info)
def setUp(self): self.domain = dm.domain(np.array([-1, -1]), np.array([1, 1]))