def construct(self, name='Sources', nsources=100, radius=1, smin=0.66545):
""" Construct nsources sources above the flux limit
"""
self.name=name
# Make more sources than we need
self.sources={}
self.sources['x'], self.sources['y']=TelUtils().uniformcircle(100*nsources, radius)
x=self.sources['x']
y=self.sources['y']
r=numpy.sqrt(x*x+y*y)
pb=numpy.exp(numpy.log(0.01)*(r/radius)**2) # Model out to 1% of PB
pb[r>radius]=0.0
f=pb*sources().randomsources(smin=smin, nsources=100*nsources)
# Avoid fields with bright sources
x=x[f<10.0*smin]
y=y[f<10.0*smin]
f=f[f<10.0*smin]
x=x[f>smin]
y=y[f>smin]
f=f[f>smin]
self.sources['x']=x[:nsources]
self.sources['y']=y[:nsources]
self.sources['flux']=f[:nsources]
print "Source fluxes = %s" % sorted(self.sources['flux'])
self.nsources=nsources
self.radius=radius
def construct(self, name='Sources', nsources=100, radius=1, smin=0.66545):
""" Construct nsources sources above the flux limit
"""
self.name = name
# Make more sources than we need
self.sources = {}
self.sources['x'], self.sources['y'] = TelUtils().uniformcircle(
100 * nsources, radius)
x = self.sources['x']
y = self.sources['y']
r = numpy.sqrt(x * x + y * y)
pb = numpy.exp(numpy.log(0.01) *
(r / radius)**2) # Model out to 1% of PB
pb[r > radius] = 0.0
f = pb * sources().randomsources(smin=smin, nsources=100 * nsources)
# Avoid fields with bright sources
x = x[f < 10.0 * smin]
y = y[f < 10.0 * smin]
f = f[f < 10.0 * smin]
x = x[f > smin]
y = y[f > smin]
f = f[f > smin]
self.sources['x'] = x[:nsources]
self.sources['y'] = y[:nsources]
self.sources['flux'] = f[:nsources]
print "Source fluxes = %s" % sorted(self.sources['flux'])
self.nsources = nsources
self.radius = radius
def get_signal(self, signal_type, file_name):
self.signal_type = signal_type
# get length of packet from the gui
self.signal_len = int(self.parent.packetLenEditor.GetLineText(0))
# TODO: setup cycles per packet in GUI
self.frequency = 5 # cycles per packet
# TODO: setup random data type in GUI
# TODO: setup random variable variance in GUI
self.rand_type = 'random'
self.file_name = file_name
self.delimiter = ','
my_signal = sources.sources(self) # instance of signals class
self.the_signal = my_signal.gen_signal()
return self.the_signal
#!/usr/bin/python2.5
from __future__ import print_function
from catalog.marc.fast_parse import *
from catalog.read_rc import read_rc
from catalog.get_ia import files
from sources import sources
import sys, os
rc = read_rc()
read_count = 10000
show_bad_records = False
for ia, name in sources(): # find which sources include '001' tag
has_001 = 0
rec_no = 0
for part, size in files(ia):
filename = rc['marc_path'] + ia + "/" + part
if not os.path.exists(filename):
continue
for data, length in read_file(open(filename)):
if rec_no == read_count:
break
rec_no += 1
if list(get_tag_lines(data, ['001'])):
has_001 += 1
elif show_bad_records:
print(data[:24])
for tag, line in get_all_tag_lines(data):
if tag.startswith('00'):
print(tag, line[:-1])
#!/usr/bin/python2.5
from __future__ import print_function
from catalog.marc.fast_parse import *
from catalog.read_rc import read_rc
from catalog.get_ia import files
from sources import sources
import sys
import os
rc = read_rc()
read_count = 10000
show_bad_records = False
for ia, name in sources(): # find which sources include '001' tag
has_001 = 0
rec_no = 0
for part, size in files(ia):
filename = rc['marc_path'] + ia + "/" + part
if not os.path.exists(filename):
continue
for data, length in read_file(open(filename)):
if rec_no == read_count:
break
rec_no += 1
if list(get_tag_lines(data, ['001'])):
has_001 += 1
elif show_bad_records:
print(data[:24])
for tag, line in get_all_tag_lines(data):
if tag.startswith('00'):
def __init__(self, master=None, thetype="IPs"):
TK.Frame.__init__( self, master, bd="1p", relief="groove", )
self.top = self.winfo_toplevel()
self.grid(pady=3, padx=0)
self.app = TK.Frame(self)
self.showsettings = TK.IntVar()
row = 0
width = 0
row2 = 0
if thetype == "IPs":
row=0
row2=0
width=31
if thetype == "URLs":
row=6
row2=6
width=30
if thetype == "SPECs":
row=12
row2=12
width=34
self.app.grid(column=4, row=row, sticky="NW")
self.thetype=thetype
self.etext = TK.StringVar()
self.label = TK.Label(self, text=thetype)
self.label.grid(column=0, row=row, sticky='NW')
self.tbox = TK.Entry(self, textvariable=self.etext, width=width)
self.tbox.grid(column=1, row=row, sticky='NW')
self.tbox.bind('<Button-3>', self.eText)
self.tbox.bind("<Return>", self.RetKey)
self.tbox.bind('<Double-Button-1>', self.eTextclear)
self.tbox["background"] = "White"
self.tbox["foreground"] = "Black"
self.button1=TK.Button(self, text="Spawn", command=lambda: self.fire(self.sources, self.values, self.trigger, self.etext))
self.button1.grid(column=2, row=row, sticky = 'NW')
if (thetype != "SPECs"):
self.button2=TK.Button(self, text="BL", command=lambda: self.processIP())
self.button2.grid(column=3, row=row, sticky = 'NW')
self.cb_settings=TK.Checkbutton(self, variable=self.showsettings, onvalue=1, offvalue=0, command=lambda: self.showSettings(row))
self.cb_settings.grid(column=0, row=row2, pady=20, sticky = 'NW')
self.a = sources.sources()
self.IPdims = self.a.IPdimensions
self.URLdims = self.a.URLdimensions
self.SPECdims = self.a.SPECIALdimensions
self.showSettings(row)
if (thetype == "IPs"):
self.sources = self.a.ip.sources
self.trigger = self.a.ip.trigger
self.links = self.a.IPLinks
self.values = [[TK.IntVar() for x in xrange(len(self.sources[:]))]]
if path.isfile(self.a.IPSettingsfile) and path.getsize(self.a.IPSettingsfile) > 0:
persist = pload(open(self.a.IPSettingsfile, "rb"))
for x in xrange(len(persist)):
self.values[0][x].set(persist[x])
for x in xrange(0, len(self.sources[:])):
self.addCheckBox(self.sources[x][0], x)
self.ip_chb_define(self.sources, self.values, self.trigger, self.etext, self.links)
self.read = self.a.IPpath
self.blread = self.a.IPbl
if (thetype == "URLs"):
self.sources = self.a.url.sources
self.trigger = self.a.url.trigger
self.links = self.a.URLLinks
self.values = [[TK.IntVar() for x in xrange(len(self.sources[:]))]]
if path.isfile(self.a.URLSettingsfile) and path.getsize(self.a.URLSettingsfile) > 0:
persist = pload(open(self.a.URLSettingsfile, "rb"))
for x in xrange(len(persist)):
self.values[0][x].set(persist[x])
for x in xrange(0, len(self.sources[:])):
self.addCheckBox(self.sources[x][0], x)
self.ip_chb_define(self.sources, self.values, self.trigger, self.etext, self.links)
self.read = self.a.URLpath
self.blread = self.a.URLbl
if (thetype == "SPECs"):
self.sources = self.a.special.sources
self.trigger = self.a.special.trigger
self.links = self.a.SPECIALLinks
self.values = TK.IntVar()
for x in xrange(0, len(self.sources[:])):
self.addRadio(self.sources[x][0], x, x+1)
self.ip_chb_define(self.sources, self.values, self.trigger, self.etext, self.links)
self.update()
def process_record(file_id, pos, length, data):
rec = index_fields(data, ['001', '010', '020', '035', '245'], check_author = False)
if not rec:
return
extra = dict((f, rec[f][0]) for f in ('title', 'lccn', 'call_number') if f in rec)
rec_id = web.insert('rec', marc_file = file_id, pos=pos, len=length, **extra)
for f in (f for f in ('isbn', 'oclc') if f in rec):
for v in rec[f]:
web.insert(f, seqname=False, rec=rec_id, value=v)
t_prev = time()
rec_no = 0
chunk = 1000
total = 32856039
for ia, name in sources():
print ia, name
for part, size in files(ia):
file_id = web.insert('files', ia=ia, part=part)
print part, size
full_part = ia + "/" + part
filename = rc['marc_path'] + full_part
if not os.path.exists(filename):
continue
pos = 0
for data, length in read_file(open(filename)):
pos += length
rec_no +=1
if rec_no % chunk == 0:
t = time() - t_prev
progress_update(rec_no, t)
if not rec:
return
extra = dict(
(f, rec[f][0]) for f in ('title', 'lccn', 'call_number') if f in rec)
rec_id = web.insert('rec', marc_file=file_id, pos=pos, len=length, **extra)
for f in (f for f in ('isbn', 'oclc') if f in rec):
for v in rec[f]:
web.insert(f, seqname=False, rec=rec_id, value=v)
t_prev = time()
rec_no = 0
chunk = 1000
total = 32856039
for ia, name in sources():
print(ia, name)
for part, size in files(ia):
file_id = web.insert('files', ia=ia, part=part)
print(part, size)
full_part = ia + "/" + part
filename = rc['marc_path'] + full_part
if not os.path.exists(filename):
continue
pos = 0
for data, length in read_file(open(filename)):
pos += length
rec_no += 1
if rec_no % chunk == 0:
t = time() - t_prev
progress_update(rec_no, t)
def __init__(self, parent, id, title, namespace, interface,
component_name, port_name, connecting_to = "component"):
self.profile_counter = 0
self.my_profile = profile.Profile()
self.numPacks = -1
self.header_count = 0
self.headers = []
self.sending_headers = False
tmp_sources_instance = sources.sources(self)
self.available_sources = tmp_sources_instance.get_sources_list()
self.start_event = threading.Event()
self._init_ctrls(parent)
self.parent = parent
self.other_tool_ref = parent
self.namespace = namespace
self.interface = interface
self.my_local_controls = None
self.component_name = component_name
self.port_name = port_name
if connecting_to == 'component':
self.setup_component_connection()
elif connecting_to == "tool":
self.setup_tool2tool_connection()
# Now Create the menu bar and items
self.mainmenu = wx.MenuBar()
menu = wx.Menu()
menu.Append(205, 'Exit', 'Enough of this already!')
self.Bind(wx.EVT_MENU, self._OnFileExit, id=205)
self.mainmenu.Append(menu, '&File')
menu = wx.Menu()
menu.Append(300, '&About', 'About this thing...')
self.Bind(wx.EVT_MENU, self._OnHelpAbout, id=300)
self.mainmenu.Append(menu, '&Help')
self.SetMenuBar(self.mainmenu)
self.Bind(wx.EVT_CLOSE, self._OnCloseWindow)
self.Show(True)
self.packetLenEditor.write('1024')
self.iFileNameEditor.write('i.dat')
self.qFileNameEditor.write('q.dat')
#set the default source
#this way if the user presses the "PushPacket" button before selecting
#a source, the source that is being displayed will be used
self.iSourceChoice.SetSelection(0)
self.qSourceChoice.SetSelection(0)
self.dataTypeChoice.SetSelection(0)
self.sendIDChoice.SetSelection(1) # 1 = No, 0 = Yes
self.timeUnitsChoice.SetSelection(1)
# start a thread that will recursively send data when the start
# button is pressed
self.is_running = True
self.send_data_thread = threading.Thread(target=self._send_data_loop)
self.send_data_thread.start()
#print('og')
og(hash, code)
#print('sitemap')
sitemap(hash, code)
#print('wordpress')
wordpress(hash, tree)
#print('nofollow')
nofollow(hash, tree)
#print('h1')
h1(hash, tree)
#print('sources')
sources(hash, result_url, result_main)
#print('robots')
robots(hash, result_main, main_hash)
#print('url_length')
url_length(hash, result_url)
#print('identical_title')
identical_title(hash, result_main)
except Exception as e:
print(e)