def __init__(self, num_threads):
self.tasks = Queue(num_threads)
for _ in range(num_threads):
Worker(self.tasks)
def __init__(self, logger, work_no, func):
self.logger = logger
self.work_no = work_no
self.func = func
self.q = Queue(10000)
self.task_no = 0
def run_download(log,
results,
abort,
title=None,
authors=None,
identifiers={},
timeout=30,
get_best_cover=False):
'''
Run the cover download, putting results into the queue :param:`results`.
Each result is a tuple of the form:
(plugin, width, height, fmt, bytes)
'''
if title == _('Unknown'):
title = None
if authors == [_('Unknown')]:
authors = None
plugins = [p for p in metadata_plugins(['cover']) if p.is_configured()]
rq = Queue()
workers = [
Worker(p,
abort,
title,
authors,
identifiers,
timeout,
rq,
get_best_cover=get_best_cover) for p in plugins
]
for w in workers:
w.start()
first_result_at = None
wait_time = msprefs['wait_after_first_cover_result']
found_results = {}
while True:
time.sleep(0.1)
try:
x = rq.get_nowait()
result = process_result(log, x)
if result is not None:
results.put(result)
found_results[result[0]] = result
if first_result_at is not None:
first_result_at = time.time()
except Empty:
pass
if not is_worker_alive(workers):
break
if first_result_at is not None and time.time(
) - first_result_at > wait_time:
log('Not waiting for any more results')
abort.set()
if abort.is_set():
break
while True:
try:
x = rq.get_nowait()
result = process_result(log, x)
if result is not None:
results.put(result)
found_results[result[0]] = result
except Empty:
break
for w in workers:
wlog = w.buf.getvalue().strip()
log('\n' + '*' * 30, w.plugin.name, 'Covers', '*' * 30)
log('Request extra headers:', w.plugin.browser.addheaders)
if w.plugin in found_results:
result = found_results[w.plugin]
log('Downloaded cover:', '%dx%d' % (result[1], result[2]))
else:
log('Failed to download valid cover')
if w.time_spent is None:
log('Download aborted')
else:
log('Took', w.time_spent, 'seconds')
if wlog:
log(wlog)
log('\n' + '*' * 80)
def __init__(self):
self._queue = Queue(maxsize=-1)
self._workers = []
self._workersToRemove = []
def ConvertToBranchQueue(self, branch_list):
branch_queue = Queue(len(branch_list))
for b, s, e in branch_list:
branch_queue.put({'base': int(b), 'start': int(s), 'end': int(e)})
return branch_queue
"--wide",
action="store_true",
default=False,
help=
"Alternate wide arrangement of widgets, for placement at bottom of 4:3 screen."
)
args = parser.parse_args()
# Some settings...
update_rate = 2 # Hz
history_size = 100 # 10 seconds at 10Hz...
history_scale = np.linspace((-1 * history_size + 1) / float(update_rate), 0,
history_size)
# Input queue
in_queue = Queue(1) # 1-element FIFO...
win = pg.GraphicsWindow()
win.setWindowTitle('FSK Demodulator Modem Statistics')
# Plot objects
ebno_plot = win.addPlot(title="Eb/No")
ppm_plot = win.addPlot(title="Sample Clock Offset")
if args.wide == False:
win.nextRow()
else:
win.resize(1024, 200)
fest_plot = pg.PlotItem() # win.addPlot(title="Tone Frequency Estimation")
eye_plot = win.addPlot(title="Eye Diagram")
# Disable auto-ranging on eye plot and fix axes for a big speedup...
spec_plot = win.addPlot(title="Spectrum")
def __init__(self):
self.ready = Queue()
self.taskmap = {}
self.exit_waiting = {}
import fnmatch
import os
from Queue import Queue
from threading import Thread
import time
import DataCollectors_Configuration
from file_operations import file_to_set, update_files
from response_getter import get_content
from helpers import string_format, url_format
from CONSTANTS import *
urls_queue = Queue()
# site = 'https://www.amazon.com/International-Shipping-Direct/b/ref=sd_allcat_full_store_dir_VisitAg?ie=UTF8&node=230659011'
def create_workers():
"""
Creates threadpool and with max number of threads mentioned in DataCollectors_Configuration file and targets work function
:return: none
"""
for _ in range(DataCollectors_Configuration.NO_OF_THEARDS):
thread = Thread(target=work)
# make the thread daemon to stop the thread when main program exits
thread.daemon = True
thread.start()
def work():
"""
Work is function which get urls from queue and calls get_products_urls(hierarchy|url)
def __init__(self, cert_db, log_key, checks=all_checks.ALL_CHECKS):
self._cert_db = cert_db
self.log_key = log_key
self._certs_queue = Queue(FLAGS.cert_db_writer_queue_size)
self._writer = None
super(CertDBCertificateReport, self).__init__(checks=checks)
def apply_mean(noisy, canny, i, j, window):
value = 0
count = 0
# left = right = up = down = True
width = noisy.shape[0]
height = noisy.shape[1]
queue = Queue()
queue.put((i, j))
while (not queue.empty() and count < window):
(k, m) = queue.get()
if canny[k][m]:
continue
if inbound(k + 1, width):
if inbound(m - 1, height) and not canny[k + 1][m - 1]:
queue.put((k + 1, m - 1))
if inbound(m, height) and not canny[k + 1][m]:
queue.put((k + 1, m))
if inbound(m + 1, height) and not canny[k + 1][m + 1]:
queue.put((k + 1, m + 1))
if inbound(k - 1, width):
if inbound(m - 1, height) and not canny[k - 1][m - 1]:
queue.put((k - 1, m - 1))
if inbound(m, height) and not canny[k - 1][m]:
queue.put((k - 1, m))
if inbound(m + 1, height) and not canny[k - 1][m + 1]:
queue.put((k - 1, m + 1))
if inbound(m + 1, height) and not canny[k][m + 1]:
queue.put((k, m + 1))
if inbound(m - 1, height) and not canny[k][m - 1]:
queue.put((k, m - 1))
value = value + noisy[k][m]
count = count + 1
print count
# while ((left or right or up or down) and count < window):
# print "i + inc: %d" % (i + inc)
# if right and (i + inc) < noisy.shape[0] and (j + inc - 1) < noisy.shape[1] and not canny[i + inc][j + inc - 1]:
# value = value + noisy[i + inc][j+inc - 1]
# count = count + 1
# else:
# right = False
# print "i - inc: %d" % (i - inc)
# if left and (i - inc) >= 0 and (j - inc + 1) < noisy.shape[1] and not canny[i - inc][j - inc + 1]:
# value = value + noisy[i + inc][j+inc - 1]
# count = count + 1
# else:
# left = False
# print "j + inc: %d" % (j + inc)
# if up and (j + inc) < noisy.shape[1] and not canny[i + inc - 1][j+inc]:
# value = value + noisy[i + inc][j+inc - 1]
# count = count + 1
# else:
# up = False
# print "j - inc: %d" % (j - inc)
# if down and (i + inc) >= 0 and not canny[i - inc + 1][j-inc]:
# value = value + noisy[i + inc][j+inc - 1]
# count = count + 1
# else:
# down = False
# inc = inc + 1
# for k in range(0,window/2):
# if (i + k) >= noisy.shape[0] or canny[i+k][j]:
# continue
# # Going forward
# for m in range(0,window/2):
# if (j + m) >= noisy.shape[1] or canny[i+k][j+m]:
# break
# # value = value + (window**2 - m*k)*float(noisy[i+k][j+m])
# value = value + float(noisy[i+k][j+m])
# # count = count + (window**2 - m*k)
# count = count + 1
# # Going backwards
# for m in range(0,window/2):
# if (j - m) >= noisy.shape[1] or canny[i+k][j-m]:
# break
# # value = value + (window**2 - m*k)*float(noisy[i+k][j-m])
# value = value + float(noisy[i+k][j-m])
# # count = count + (window**2 - m*k)
# count = count + 1
# for k in range(0,window/2):
# if (i - k) >= noisy.shape[0] or canny[i-k][j]:
# continue
# # Going forward
# for m in range(0,window/2):
# if (j + m) >= noisy.shape[1] or canny[i-k][j+m]:
# break
# # value = value + (window**2 - m*k)*float(noisy[i-k][j+m])
# value = value + float(noisy[i-k][j+m])
# # count = count + (window**2 - m*k)
# count = count + 1
# # Going backwards
# for m in range(0,window/2):
# if (j - m) >= noisy.shape[1] or canny[i-k][j-m]:
# break
# # value = value + (window**2 - m*k)*float(noisy[i-k][j-m])
# value = value + float(noisy[i-k][j-m])
# # count = count + (window**2 - m*k)
# count = count + 1
# if count is not 16:
# print count
return value / float(count)
MSG_PROLOGUE = "0" * 10
MSG_EPILOGUE = "1" * 10
inp = alsaaudio.PCM(alsaaudio.PCM_CAPTURE)
inp.setchannels(2)
inp.setrate(SAMPLES_PER_SECOND)
inp.setformat(alsaaudio.PCM_FORMAT_S16_LE)
inp.setperiodsize(PERIOD)
outp = alsaaudio.PCM()
outp.setchannels(2)
outp.setrate(SAMPLES_PER_SECOND)
outp.setformat(alsaaudio.PCM_FORMAT_S16_LE)
outp.setperiodsize(PERIOD)
send_queue = Queue()
recv_queue = Queue()
if TUN_ENABLED:
tun = pytun.TunTapDevice(flags=pytun.IFF_TUN | pytun.IFF_NO_PI)
tun.addr = {"daemon": "192.168.133.7", "elias": "192.168.133.8"}[host]
tun.dstaddr = {"daemon": "192.168.133.8", "elias": "192.168.133.7"}[host]
tun.netmask = "255.255.255.0"
tun.mtu = MTU
tun.up()
else:
tun = None
the_end = threading.Event()
def __init__(self):
self.valid_proxy_queue = Queue()
def __init__(self, sheet_lock, sheet_url):
self.sheet_lock = sheet_lock
self.sheet_url = sheet_url
self.queue = Queue()
self.pushed = 0
self.popped = 0
# User inputs
target_path = sys.argv[1].rstrip("/")
n_cpus = int(sys.argv[2])
# Docking parameters
receptor = target_path + "/receptor.pdbqt"
crystal_ligand = target_path + "/crystal_ligand.mol2"
actives = glob.glob(target_path + "/actives/*.pdbqt")
decoys = glob.glob(target_path + "/decoys/*.pdbqt")
ligands = actives + decoys
num_modes = 12
start_time = time.time()
# Initialize queue
q = Queue(maxsize=n_cpus)
start_time = time.time()
while len(ligands) != 0:
while not q.full():
ligand = ligands.pop()
ligname = path.splitext(ligand)[0]
# Start subprocess
p = Popen([SMINA_PATH, "-r", receptor, "-l", ligand, "--autobox_ligand",
crystal_ligand, "--num_modes", str(num_modes), "-o",
ligname + "_out.pdbqt", "--cpu", "1"])
# Add to queue
q.put(p)
# Get first item and check if it finished
top = q.get()
poll = top.poll()
def __init__(self):
self.queue = Queue()
import sys
from Queue import Queue
import sip_parser
from sip_transaction_manager import SIPTransactionManager
from sip_agent import SIPCancel
from sip_agent import SIPAck
from sip_agent import SIPAgent
from sip_agent import SIPRegister
from sip_agent import SIPInvite
response_q = Queue(0)
request_q = Queue(0)
sip_agent = SIPAgent()
register = SIPRegister()
SIPTransactionManager(request_q, response_q).start()
register_dict = {
sip_parser.r_SEND: (register.process, {
sip_parser.r_4XX: (register.process, {
sip_parser.r_2XX: (sip_agent.require, None),
sip_parser.r_1XX: (None, None)
}),
sip_parser.r_1XX: (None, None),
sip_parser.r_2XX: (sip_agent.require, None),
def create(dataset,
dataset_name,
output_directory,
num_shards,
num_threads,
shuffle=True):
"""Create the tfrecord files to be used to train or test a model.
Args:
dataset : [{
"filename" : <REQUIRED: path to the image file>,
"id" : <REQUIRED: id of the image>,
"class" : {
"label" : <[0, num_classes)>,
"text" : <text description of class>
},
"object" : {
"bbox" : {
"xmin" : [],
"xmax" : [],
"ymin" : [],
"ymax" : [],
"label" : []
}
}
}]
dataset_name: a name for the dataset
output_directory: path to a directory to write the tfrecord files
num_shards: the number of tfrecord files to create
num_threads: the number of threads to use
shuffle : bool, should the image examples be shuffled or not prior to creating the tfrecords.
Returns:
list : a list of image examples that failed to process.
"""
# Images in the tfrecords set must be shuffled properly
if shuffle:
random.shuffle(dataset)
# Break all images into batches with a [ranges[i][0], ranges[i][1]].
spacing = np.linspace(0, len(dataset), num_threads + 1).astype(np.int)
ranges = []
threads = []
for i in xrange(len(spacing) - 1):
ranges.append([spacing[i], spacing[i + 1]])
# Launch a thread for each batch.
print('Launching %d threads for spacings: %s' % (num_threads, ranges))
sys.stdout.flush()
# Create a mechanism for monitoring when all threads are finished.
coord = tf.train.Coordinator()
# Create a generic TensorFlow-based utility for converting all image codings.
coder = ImageCoder()
# A Queue to hold the image examples that fail to process.
error_queue = Queue()
threads = []
for thread_index in xrange(len(ranges)):
args = (coder, thread_index, ranges, dataset_name, output_directory,
dataset, num_shards, error_queue)
t = threading.Thread(target=_process_image_files_batch, args=args)
t.start()
threads.append(t)
# Wait for all the threads to terminate.
coord.join(threads)
print('%s: Finished writing all %d images in data set.' %
(datetime.now(), len(dataset)))
# Collect the errors
errors = []
while not error_queue.empty():
errors.append(error_queue.get())
print('%d examples failed.' % (len(errors), ))
return errors
def __init__(self, listener):
Thread.__init__(self)
self.daemon = True
self.listener, self.queue = listener, Queue()
self._run = True
self.start()
def customFunc(x):
#Write your awesome processing code here
return
def do_stuff(q):
while True:
x = q.get()
if not getFound():
customFunc(x)
if (x == 150000):
setFound()
q.task_done()
q = Queue(maxsize=0)
try:
numThreads = int(sys.argv[1])
print "Using %s Threads " % numThreads
except:
numThreads = 4
print "Deafult Threads ", numThreads
t1 = (time.time())
print "Starting Tests - Time is: %s\n" % t1
for i in range(numThreads):
t = Thread(target=do_stuff, args=(q, ))
t.setDaemon(True)
t.start()
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""
MAX_CONTEXT = 3
chat_history = {}
chat_timing = {} # just for testing response time
# Queues
processing_msg_queue = Queue()
outgoing_msg_queue = Queue() #multiprocessing.JoinableQueue()
# Pipes
# parent to bot caller
PARENT_PIPE = 'ipc:///tmp/parent_push.pipe'
# bot to parent caller
PARENT_PULL_PIPE = 'ipc:///tmp/parent_pull.pipe'
class ChatState:
START = 0 # when we received `/start`
END = 1 # when we received `/end`
CHATTING = 2 # all other times
CONTROL = 3 # for control msgs
so.send_toRemote("on loop")
a = 1
"""
print("Within loop")
so.send_toRemote("within loop")
print("Received message: ", data)
#prediction = neural_network_model(x)
#init = tf.global_variables_initializer()
q = Queue()
t1 = threading.Thread(target = sensor, args = (q,))
t2 = threading.Thread(target = modelCar, args = (q,))
t1.start()
t2.start()
t1.join()
t2.join()
print("Program ends here")
GPIO.cleanup()
label1[cnt, int(fn2[1])] = 1
data[cnt,:,:,:]=im
cnt = cnt + 1
except Exception:
pass
if vbx>0:
print("There are " + str(vbx) + " times that exception is happen.")
batch_num = batch_num + 1
return data, label1
#async load data
data_queue = Queue(8)
def load_data():
while True:
if Queue.qsize(data_queue)<8:
#print 'loading next batch...'+str(Queue.qsize(data_queue))
x,y = imagenet_batch(batch_size)
data_queue.put((x,y))
time.sleep(1)
def batch_norm(x, n_out, scope='bn'):
"""
Batch normalization on convolutional maps.
def test2_PersistentDB(self):
numThreads = 3
persist = PersistentDB(dbapi)
from Queue import Queue, Empty
queryQueue, resultQueue = [], []
for i in range(numThreads):
queryQueue.append(Queue(1))
resultQueue.append(Queue(1))
def runQueries(i):
this_db = persist.connection()
while 1:
try:
q = queryQueue[i].get(1, 1)
except Empty:
q = None
if not q: break
db = persist.connection()
if db != this_db:
r = 'error - not persistent'
else:
if q == 'ping':
r = 'ok - thread alive'
elif q == 'close':
db.close()
r = 'ok - connection closed'
else:
cursor = db.cursor()
cursor.execute(q)
r = cursor.fetchone()
cursor.close()
r = '%d(%d): %s' % (i, db._usage, r)
resultQueue[i].put(r, 1, 0.1)
db.close()
from threading import Thread
threads = []
for i in range(numThreads):
thread = Thread(target=runQueries, args=(i,))
threads.append(thread)
thread.start()
for i in range(numThreads):
queryQueue[i].put('ping', 1, 0.1)
for i in range(numThreads):
r = resultQueue[i].get(1, 0.1)
self.assertEqual(r, '%d(0): ok - thread alive' % i)
self.assert_(threads[i].isAlive())
for i in range(numThreads):
for j in range(i + 1):
queryQueue[i].put('select test%d' % j, 1, 0.1)
r = resultQueue[i].get(1, 0.1)
self.assertEqual(r, '%d(%d): test%d' % (i, j + 1, j))
queryQueue[1].put('select test4', 1, 0.1)
r = resultQueue[1].get(1, 0.1)
self.assertEqual(r, '1(3): test4')
queryQueue[1].put('close', 1, 0.1)
r = resultQueue[1].get(1, 0.1)
self.assertEqual(r, '1(0): ok - connection closed')
for j in range(2):
queryQueue[1].put('select test%d' % j, 1, 0.1)
r = resultQueue[1].get(1, 0.1)
self.assertEqual(r, '1(%d): test%d' % (j + 1, j))
for i in range(numThreads):
self.assert_(threads[i].isAlive())
queryQueue[i].put('ping', 1, 0.1)
for i in range(numThreads):
r = resultQueue[i].get(1, 0.1)
self.assertEqual(r, '%d(%d): ok - thread alive' % (i, i + 1))
self.assert_(threads[i].isAlive())
for i in range(numThreads):
queryQueue[i].put(None, 1, 1)
def main(argv=None):
global stopMe, options
myrootpath = os.path.dirname(os.path.realpath(__file__)) + "/"
print("START DEAMON")
(options, args) = cli_parser(argv)
write_pid(options.pid_path)
LOG_FILENAME = myrootpath + '../../../../log/duibridge_daemon'
formatter = logging.Formatter(
'%(threadName)s-%(asctime)s| %(levelname)s | %(lineno)d | %(message)s')
'''filehandler = logging.FileHandler(LOG_FILENAME)
filehandler.setFormatter(formatter)
logger.addHandler(filehandler)
'''
console = logging.StreamHandler()
console.setFormatter(formatter)
logger.addHandler(console)
logger.setLevel(logging.DEBUG)
sys.stderr = open(LOG_FILENAME + "_stderr", 'a', 1)
options.loglevel.upper()
logger.setLevel(logging.getLevelName(options.loglevel.upper()))
logger.info("# duiBridged - duinode bridge for Jeedom # loglevel=" +
options.loglevel)
options.pin_config = {}
all_pins_decode = None
##arduino_id="A1"
conf_pins_modif_time = os.stat(options.config_pins_path).st_mtime
logger.info("Config last modif time" + str(conf_pins_modif_time))
for arduino_num in range(1, 2):
arduino_id = "A" + str(arduino_num)
options.pin_config.update({
arduino_id:
Pin_Config(arduino_id, options.config_pins_path,
options.config_ports_path)
})
pins_decode = options.pin_config[arduino_id].load_pin_config(
all_pins_decode)
if not pins_decode == None:
''' config found for this Arduino, json array saved for the next round '''
all_pins_decode = pins_decode
''' hope that the port is also defined '''
options.pin_config[arduino_id].load_port_config()
options.nodes = {}
### rootNode = options.pin_config[arduino_id].rootNode ## TODO manage several arduino
options.to_arduino_queues = {arduino_id: Queue()}
options.from_arduino_queues = {arduino_id: Queue()}
mqttc1 = MQTT_Client()
mqttc1.run(arduino_id, "localhost",
options.to_arduino_queues[arduino_id])
if options.pin_config[arduino_id].port == "bridge":
arduidom_mqtt = MQTT_Arduidom()
arduidom_queue = Queue()
arduidom_mqtt.run(arduino_id, "localhost", arduidom_queue)
aNode = Arduidom_node(options.pin_config[arduino_id].port,
options.to_arduino_queues[arduino_id],
arduino_id,
options.from_arduino_queues[arduino_id],
mqttc1, arduidom_queue, arduidom_mqtt)
arduidom_mqtt.set_topic("duitest/abridge/toarduino",
"duitest/abridge/fromarduino")
cp_cmd = None
else:
aNode = Arduino_Node(options.pin_config[arduino_id].port,
options.to_arduino_queues[arduino_id],
arduino_id,
options.from_arduino_queues[arduino_id],
mqttc1)
cp_cmd = options.pin_config[arduino_id].get_pin_conf_cmd()
options.nodes.update({arduino_id: aNode})
if not cp_cmd == None:
request = Arduino_Request(cp_cmd, "CP_OK")
options.to_arduino_queues[arduino_id].put(request)
## subscribe to digital topics if any
pin_list = options.pin_config[arduino_id].all_pins.values()
for pin in pin_list:
##(mode, topic) = m_t
if not (pin.mode in Pin_def.mode_status):
mqttc1.subscribe(pin.topic)
##logger.debug('subscribe :'+pin.topic)
##else:
##logger.debug('not subscribe {} {}:'.format(pin.mode, pin.topic))
''' subscribe to radio topics '''
topics = options.pin_config[arduino_id].t_radio_vpins.keys()
mqttc1.subscribe_topics(topics)
else:
logger.info("config not found for " + arduino_id)
count = 0
while True:
time.sleep(0.1)
count += 1
if not options.from_arduino_queues[arduino_id].empty():
mess = str(options.from_arduino_queues[arduino_id].get(False))
logger.debug("from_arduino_queues:" + mess)
if '>>' in mess[:6]:
(pin, value) = mess.split(">>")
value = value.replace("<<", '')
if value[-3:] == '.00':
value = value[:(len(value) - 3)]
logger.debug(str(pin) + " " + str(value))
send_to_topic(arduino_id, pin, value, mqttc1)
elif 'DHT' in mess[:6]:
''' Manage DHT Values '''
line = mess[4:-1] ### remove DHT: in begining and the last ;
dht_values = line.split(';')
for pinnumber in range(0, len(dht_values)):
if "na" not in dht_values[pinnumber]:
logger.debug("DHT: {} = {}".format(
501 + pinnumber, dht_values[pinnumber]))
send_to_topic(arduino_id, 501 + pinnumber,
dht_values[pinnumber], mqttc1)
options.from_arduino_queues[arduino_id].task_done()
else:
if count > 20:
count = 0
new_conf_pins_modif_time = os.stat(
options.config_pins_path).st_mtime
if (new_conf_pins_modif_time != conf_pins_modif_time):
conf_pins_modif_time = new_conf_pins_modif_time
logger.info(" Config pins changed")
all_pins_decode = None
for arduino_num in range(1, 2):
arduino_id = "A" + str(arduino_num)
##options.pin_config.update({arduino_id:Pin_Config(arduino_id, options.config_pins_path, options.config_ports_path)})
pins_decode = options.pin_config[
arduino_id].reload_pin_config(all_pins_decode)
if not pins_decode == None:
''' config found for this Arduino, json array saved for the next round '''
all_pins_decode = pins_decode
logger.info(" Config pins reloaded ")
logger.info("THE END")
import urllib2
NUMBER_OF_CRAWLERS = 20
NUMBER_OF_PARSERS = 5
# Make Temporary Directory
BASE_DIR = os.path.dirname(__file__)
TEMP_DIR = os.path.join(BASE_DIR, 'temp')
if not os.path.exists(TEMP_DIR):
os.mkdir(TEMP_DIR)
_stdout_lock = threading.Lock()
_url_queue = Queue()
_content_queue = Queue()
_url_history = {}
class Crawler(threading.Thread):
def __init__(self, baseUrl, lock, urlQueue, contentQueue):
super(Crawler, self).__init__()
self.baseUrl = baseUrl
self.baseHostname = urlparse(self.baseUrl).hostname
self.lock = lock
self.urlQueue = urlQueue
self.contentQueue = contentQueue
def __init__(self):
self.queue = Queue()
self.waiting = []
def __init__(self, checks=all_checks.ALL_CHECKS, queue_size=None):
self.reset()
self.checks = checks
self._jobs = Queue(queue_size or FLAGS.reporter_queue_size)
self._pool = None
self._writing_handler = None
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sun Dec 13 16:46:28 2020
@author: Anjali Venugopal
@secondary-author: Abhiram Shibu
"""
from Queue import Queue
testpass=True
myQueue = Queue(10)
if(myQueue.dequeue()!=None):
testpass=False
if(myQueue.isEmpty()!=True):
testpass=False
if(myQueue.isFull()!=False):
testpass=False
if(myQueue.peek()!=None):
testpass=False
myQueue.enqueue(1)
myQueue.enqueue(2)
myQueue.enqueue(3)
if(myQueue.dequeue()!=1):
testpass=False
if(myQueue.dequeue()!=2):
testpass=False
if(myQueue.dequeue()!=3):
testpass=False
if(myQueue.dequeue()!=None):
testpass=False
from utility import *
from selenium import webdriver
from selenium.webdriver.common.keys import Keys
#from selenium.webdriver.common.proxy import *
import httpagentparser
import json, io, threading
START_QUEUE = 0
END_QUEUE = 2000000
num_threads = 100
num_skipped = 0
base_url = "https://mbasic.facebook.com/"
usernames = []
queue = Queue(maxsize=0)
useragent = UserAgent()
counter = START_QUEUE
class ThreadWorker(threading.Thread):
def __init__(self, queue):
threading.Thread.__init__(self)
self.queue = queue
options, srv = self.getCapability()
self.driver = webdriver.PhantomJS("./phantomjs",
desired_capabilities=options,
service_args=srv)
def run(self):
while self.queue.qsize() != 0:
'correction_secs':
sorted(self.recentCorrections)[len(
self.recentCorrections) // 2],
}
]), address)
elif message[0] == 'terminate':
break
sock.close()
if __name__ == '__main__':
if len(sys.argv) == 2 and sys.argv[1].startswith('-r'):
# Receiver
triggerQ = Queue()
def triggerCallback(info):
triggerQ.put(info)
def printQ():
while 1:
info = triggerQ.get()
print info
triggerQ.task_done()
triggerPrinter = threading.Thread(target=printQ)
triggerPrinter.daemon = True
triggerPrinter.start()
receiver = MultiCastReceiver(triggerCallback)
