def collect(self):
self.logger.info("Collecting following edges for node: %s" % (self.node))
data=""
try:
conn = httplib.HTTPConnection(self.ip)
url = "/website-crawler/following?&cursor_count=%s&screen_name=%s" %\
(self.config[ConfigKeys.SNS][ConfigKeys.CURSOR_COUNT],
self.node[MongoDB.TWITTER_SNAME_ID_KEY])
self.logger.info("Connecting to: %s%s" % (self.ip, url))
conn.request("GET", url)
r = conn.getresponse()
data = r.read()
conn.close()
except socket.error:
self.logger.\
error("Socket error when collecting edges for node: %s" %\
self.node)
return False
except HTTPException:
self.logger.\
error("HTTP error when collecting edges for node: %s" %\
self.node)
return False
edges = set([sname.strip() \
for sname in data.split(",") if len(sname.strip()) > 0])
Processor.processEdges(self.config,
self.logger, self.db,
self.node, self.level,
ConfigKeys.OUT_EDGES_KEY, edges, MongoDB.TWITTER_SNAME_ID_KEY)
return True
def run_processor(g=g):
from processor import Processor
processor = Processor(projectdb=g.projectdb,
inqueue=g.fetcher2processor, status_queue=g.status_queue,
newtask_queue=g.newtask_queue, result_queue=g.processor2result)
processor.run()
class Router(object):
def __init__(self):
self.processor = Processor()
def __del__(self):
self.close()
def close(self):
self.processor.close()
def dispatch(self, request_handler):
# start = time.time()
db().open_session()
db_rcd().open_session()
try:
token = self.parse_token(request_handler.request)
msg_list = self.parse_msg(request_handler.request)
val_list = self.handle_msg(token, msg_list,
request_handler)
response = self.parse_response(val_list)
return response
except PiscesException, ex:
db_rcd().rollback()
db().rollback()
log_root().exception('uncaught pisces except: ' + ex.msg)
finally:
class GUI(Tk):
"represents GUI"
def __init__(self):
super().__init__()
self.title("CM Manager " + ".".join(version()))
self.option_add("*tearOff", FALSE)
self.resizable(FALSE, FALSE)
'''
# used when size of the window is changed for placeWindow arguments
self.after(250, lambda: print(self.winfo_width()))
self.after(250, lambda: print(self.winfo_height()))
'''
placeWindow(self, 1010, 834)
# notebook
self.selectFunction = ttk.Notebook(self)
self.selectFunction.grid()
# FileStorage is associated with the Notebook
self.selectFunction.fileStorage = FileStorage()
self.processor = Processor(self.selectFunction)
self.explorer = Explorer(self.selectFunction)
self.controller = Controller(self.selectFunction)
notepageWidth = 20
self.selectFunction.add(self.processor, text = "{:^{}}".format("Process", notepageWidth))
self.selectFunction.add(self.explorer, text = "{:^{}}".format("Explore", notepageWidth))
self.selectFunction.add(self.controller, text = "{:^{}}".format("Control", notepageWidth))
self.selectFunction.bind("<<NotebookTabChanged>>", lambda e: self.checkProcessing(e))
# menu
self["menu"] = MenuCM(self)
if not optionGet("Developer", False, 'bool'):
self.protocol("WM_DELETE_WINDOW", self.closeFun)
self.mainloop()
def closeFun(self):
"ask for saving files on exit"
if doesFileStorageRequiresSave(self):
answ = messagebox.askyesno(message = "Do you want to save files before leaving?",
icon = "question", title = "Save files?")
if answ:
saveFileStorage(self)
self.destroy()
def checkProcessing(self, event):
"""checks whether it is possible for processor and controller to process files and change
states of buttons accordingly"""
self.processor.checkProcessing()
self.controller.checkProcessing()
self.explorer.checkProcessing()
def __init__(self, config, shared):
Processor.__init__(self)
# monitoring
self.avg_time = 0,0,0
self.time_ref = time.time()
self.shared = shared
self.config = config
self.up_to_date = False
self.watch_lock = threading.Lock()
self.watch_blocks = []
self.watch_headers = []
self.watched_addresses = {}
self.history_cache = {}
self.merkle_cache = {}
self.max_cache_size = 100000
self.chunk_cache = {}
self.cache_lock = threading.Lock()
self.headers_data = ''
self.headers_path = config.get('leveldb', 'path')
self.mempool_fees = {}
self.mempool_values = {}
self.mempool_addresses = {}
self.mempool_hist = {} # addr -> (txid, delta)
self.mempool_unconfirmed = {} # txid -> set of unconfirmed inputs
self.mempool_hashes = set()
self.mempool_lock = threading.Lock()
self.address_queue = Queue()
try:
self.test_reorgs = config.getboolean('leveldb', 'test_reorgs') # simulate random blockchain reorgs
except:
self.test_reorgs = False
self.storage = Storage(config, shared, self.test_reorgs)
self.bitcoind_url = 'http://%s:%s@%s:%s/' % (
config.get('bitcoind', 'bitcoind_user'),
config.get('bitcoind', 'bitcoind_password'),
config.get('bitcoind', 'bitcoind_host'),
config.get('bitcoind', 'bitcoind_port'))
self.sent_height = 0
self.sent_header = None
# catch_up headers
self.init_headers(self.storage.height)
# start catch_up thread
if config.getboolean('leveldb', 'profiler'):
filename = os.path.join(config.get('leveldb', 'path'), 'profile')
print_log('profiled thread', filename)
self.blockchain_thread = ProfiledThread(filename, target = self.do_catch_up)
else:
self.blockchain_thread = threading.Thread(target = self.do_catch_up)
self.blockchain_thread.start()
def test_move(self):
processor = Processor()
worker = processor.get_worker("/", "/users/system")
worker.move("users")
self.assertEqual("/users", worker.current_item.handle.path)
worker.move("system")
self.assertEqual("/users/system", worker.current_item.handle.path)
self.assertRaises(Exception, worker.move, "not there")
def install(self):
""" Install assets in target dir. """
processor = Processor(
cache_dir=self.config['CACHE_DIR'],
target_dir=self.config['TARGET_DIR']
)
processor.resolve_asset_defs(self.target_defs)
print >> sys.stderr, "Install finished."
def __init__(self, config, shared):
Processor.__init__(self)
self.active_chain = chainparams.get_active_chain()
self.mtimes = {} # monitoring
self.shared = shared
self.config = config
self.up_to_date = False
self.watch_lock = threading.Lock()
self.watch_blocks = []
self.watch_headers = []
self.watched_addresses = {}
self.block_cache = OrderedDict()
try:
self.block_cache_size = config.getint('server', 'block_cache_size')
except ConfigParser.NoOptionError:
self.block_cache_size = default_block_cache_size
self.history_cache = {}
self.max_cache_size = 100000
self.chunk_cache = {}
self.cache_lock = threading.Lock()
self.headers_data = ''
self.headers_path = config.get('leveldb', 'path')
self.mempool_values = {}
self.mempool_addresses = {}
self.mempool_hist = {}
self.mempool_hashes = set([])
self.mempool_lock = threading.Lock()
self.address_queue = Queue()
try:
self.test_reorgs = config.getboolean('leveldb', 'test_reorgs') # simulate random blockchain reorgs
except:
self.test_reorgs = False
self.storage = Storage(config, shared, self.test_reorgs)
self.dblock = threading.Lock()
self.bitcoind_url = 'http://%s:%s@%s:%s/' % (
config.get('bitcoind', 'bitcoind_user'),
config.get('bitcoind', 'bitcoind_password'),
config.get('bitcoind', 'bitcoind_host'),
config.get('bitcoind', 'bitcoind_port'))
self.sent_height = 0
self.sent_header = None
# catch_up headers
self.init_headers(self.storage.height)
self.blockchain_thread = threading.Thread(target = self.do_catch_up)
self.blockchain_thread.start()
def ingest_file(file):
pitch = Processor()
i = 0
with open(file) as f:
content = f.readlines()
for line in content:
pitch.parse(line)
i += 1
return pitch.print_top_symbols()
def __init__(self, config):
Processor.__init__(self)
self.daemon = True
self.banner = config.get('server', 'banner')
if config.get('server', 'irc') == 'yes':
self.irc = IrcThread(self, config)
else:
self.irc = None
def test_find_or_create(self):
processor = Processor()
worker = processor.get_worker("/", "/users/system")
self.assertIsNone(worker.find("floop").item_id)
handle1 = worker.find_or_create("floop", "folder")
handle2 = worker.find_or_create("floop", "folder")
self.assertEqual("/floop", handle1.path)
self.assertEqual("/floop", handle2.path)
self.assertTrue(handle1.item_id == handle2.item_id)
def main():
""" Main function """
p = Processor()
p.process(sys.argv[1],
output=config.OUTPUT_FILE,
estimate_curpair=config.ESTIMATE_CURPAIR,
feature_curpairs=config.FEATURE_CURPAIRS,
window_size=config.WINDOW_SIZE,
timeslot_len=config.TIMESLOT_LEN)
def ncd_loop(doInit, dlThreadNum):
ndutil.setTimezone()
#read config
cnfManager = CnfManager()
cnfManager.load('./ndc.cnf')
cnfData = cnfManager.getCnfData()
#check dirs
ndutil.enableDir(cnfData['dirWorking'])
ndutil.enableDir(cnfData['dirStore'])
#ndlcom
logger = Logger('nDroid-Crawler', cnfData['ndlComHost'], cnfData['ndlComPort'])
logger.logger('Initiating')
dbManager = DbManager(cnfData['dbHost'], cnfData['dbUser'], cnfData['dbPass'], cnfData['dbName'])
if doInit:
dbManager.create_table()
os.system('rm -f %s/*' % cnfData['dirWorking'])
os.system('rm -f %s/*' % cnfData['dirStore'])
#logger.logger('Customizing Spiders')
#spiderGenerator = SpiderGenerator('template', 'spider/spiders')
#for spider in cnfData['spiders']:
# spiderGenerator.gen_spider(spider, cnfData[spider]['startPage'], cnfData[spider]['stopPage'])
rpQueue = Queue()
pdQueue = Queue()
dpQueue = Queue()
pdLock = threading.Lock()
rpcMonitor = RpcMonitor(logger, cnfData['rpcPort'], cnfData['rpcAuth'], 'RpcMonitor')
rpcMonitor.setGlobalInfo(ndutil.getCurrentTimeStr(), 'Standalone', dlThreadNum)
rpcMonitor.setDownloadTotal(pdQueue.qsize())
rpcMonitor.setPdQueueSize(pdQueue.qsize())
botScheduler = BotScheduler(logger, rpcMonitor, cnfData['spiders'], cnfData['spiderCnfs'], 'BotScheduler')
receiver = Receiver(logger, rpcMonitor, rpQueue, cnfData['receiverPort'], 'Receiver')
preProcessor = PreProcessor(logger, rpcMonitor, rpQueue, pdQueue, pdLock, dbManager, 'PreProcessor')
downloader = Downloader([logger, rpcMonitor, pdQueue, dpQueue, pdLock, dlThreadNum, cnfData['dirWorking']], 'Downloader')
processor = Processor(logger, rpcMonitor, dpQueue, pdLock, pdQueue, dbManager, cnfData['dirWorking'], cnfData['dirStore'], 'Processor')
logger.logger('Starting Threads')
rpcMonitor.start()
botScheduler.start()
receiver.start()
preProcessor.start()
downloader.start()
processor.start()
processor.join()
downloader.join()
preProcessor.join()
receiver.join()
botScheduler.join()
rpcMonitor.join()
def __init__(self, config):
Processor.__init__(self)
cache = HistoryCache()
self.backend = Backend()
monitor = MonitorAddress(self, cache, self.backend)
self.numblocks_subscribe = NumblocksSubscribe(self.backend, self)
self.address_get_history = AddressGetHistory(self.backend, self)
self.address_subscribe = \
AddressSubscribe(self.backend, self, cache, monitor)
def __init__(self, config):
Processor.__init__(self)
self.daemon = True
self.banner = config.get("server", "banner")
if config.get("server", "irc") == "yes":
self.irc = IrcThread(self, config)
else:
self.irc = None
def terminal_input():
continue_input = True
pitch = Processor()
while continue_input:
op = input("Enter operation [Q to quit]: ")
if op == "Q":
return
pitch.parse(op)
pitch.print_top_symbols()
def __init__(self, config, shared):
Processor.__init__(self)
self.mtimes = {} # monitoring
self.shared = shared
self.config = config
self.up_to_date = False
self.watch_lock = threading.Lock()
self.watch_blocks = []
self.watch_headers = []
self.watched_addresses = {}
self.history_cache = {}
self.max_cache_size = 100000
self.chunk_cache = {}
self.cache_lock = threading.Lock()
self.headers_data = ''
self.headers_path = config.get('leveldb', 'path')
self.mempool_values = {}
self.mempool_addresses = {}
self.mempool_hist = {}
self.mempool_hashes = set([])
self.mempool_lock = threading.Lock()
self.address_queue = Queue()
try:
self.test_reorgs = config.getboolean('leveldb', 'test_reorgs') # simulate random blockchain reorgs
except:
self.test_reorgs = False
self.storage = Storage(config, shared, self.test_reorgs)
self.dblock = threading.Lock()
self.bitcoind_rpc_ssl = False
if config.has_option('bitcoind', 'bitcoind_rpc_ssl') and config.get('bitcoind', 'bitcoind_rpc_ssl') == '1':
self.bitcoind_rpc_ssl = True
self.bitcoind_url = '%s://%s:%s@%s:%s/' % (
'https' if self.bitcoind_rpc_ssl else 'http',
config.get('bitcoind', 'bitcoind_user'),
config.get('bitcoind', 'bitcoind_password'),
config.get('bitcoind', 'bitcoind_host'),
config.get('bitcoind', 'bitcoind_port'))
self.sent_height = 0
self.sent_header = None
# catch_up headers
self.init_headers(self.storage.height)
self.blockchain_thread = threading.Thread(target = self.do_catch_up)
self.blockchain_thread.start()
def update(self):
""" Update assets. Removes and then re-installs assets. """
processor = Processor(
cache_dir=self.config['CACHE_DIR'],
target_dir=self.config['TARGET_DIR']
)
print self.target_defs
for asset_id, asset_def in self.target_defs.items():
self.remove_asset(asset_id)
processor.resolve_asset_defs({asset_id: asset_def})
print >> sys.stderr, "Update finished."
def __init__(self, config):
Processor.__init__(self)
self.store = AbeStore(config)
self.block_number = -1
self.watched_addresses = []
# catch_up first
n = self.store.main_iteration()
print "blockchain: %d blocks"%n
threading.Timer(10, self.run_store_iteration).start()
def worker():
# Print start time
Log.log.time("Start")
sql_script = 'adres-tabel.sql'
# Extracts any data from any source files/dirs/zips/xml/csv etc
Database().log_actie('start_maak_adressen', sql_script)
processor = Processor()
processor.processSQLScript(sql_script)
Database().log_actie('stop_maak_adressen', sql_script)
# Print end time
Log.log.time("End")
def main():
try:
source_code_file_name = sys.argv[1]
except:
source_code_file_name = 'human-code.txt'
# source_code_file_name = 'human-code-2.txt'
code_file = file(source_code_file_name, 'r')
initial_values_file = file('./initial_values_matrix.txt', 'r')
# initial_values_file = file('./initial_values_fibo.txt', 'r')
processor = Processor(code_file, initial_values_file)
processor.run()
def run(self):
processor = Processor()
config = Configurator().read()
processor.config = config
to = float(config.pollTimeOut)
print "Offwind processing service started. Polling queue... timeout " + config.pollTimeOut
while 1==1:
print datetime.utcnow()
sleep(to)
processor.Do()
print "Exiting program... Bye!"
def __init__(self, config, shared):
Processor.__init__(self)
self.store = AbeStore(config)
self.watched_addresses = []
self.shared = shared
# catch_up first
self.block_header, time_catch_up, time_mempool, n = self.store.main_iteration()
self.block_number = self.block_header.get('block_height')
print_log("blockchain: %d blocks" % self.block_number)
threading.Timer(10, self.run_store_iteration).start()
def start(self):
self.logger.info("Collecting followers for node: %s" % (self.node))
oauth = OAuth1(self.app["consumerKey"],
self.app["consumerSecret"],
self.app["accessToken"],
self.app["accessTokenSecret"])
url="%s/followers/ids.json?stringify_ids=true&user_id=%s&count=%s&cursor=%s" %\
(self.BASE_URL,
self.node[MongoDB.TWITTER_ID_KEY],
self.config[ConfigKeys.MULTITHREADING][ConfigKeys.FOLLOWERS_REQUEST_SIZE],
self.cursor)
self.logger.info("Connecting to: %s" % (url))
resp = requests.get(url, auth=oauth)
self.logger.info("HEADERS RECEIVED for node %s: %s" % (self.node, resp.headers))
limitRemaining = 0
nextCursor = self.cursor
if 'x-rate-limit-remaining' in resp.headers:
limitRemaining = int(resp.headers['x-rate-limit-remaining'])
else:
self.logger.error("INVALID RESPONSE: for node %s: %s" %\
(self.node, resp))
return (0, 0)
data = resp.content.decode('utf-8')
dataO = {}
try:
dataO = json.loads(data)
except ValueError:
self.logger.error("INVALID RESPONSE: json not found for node %s: %s" %\
(self.node, data))
return (0, 0)
if "ids" in dataO:
edges = dataO["ids"]
nextCursor = dataO["next_cursor"]
Processor.processEdges(self.config,
self.logger, self.db,
self.node, self.level,
ConfigKeys.IN_EDGES_KEY, edges, MongoDB.TWITTER_ID_KEY)
return (limitRemaining, nextCursor)
else:
self.logger.info("IDS NOT FOUND (LIMIT MIGHT BE EXCEEDED): for node %s: %s" %\
(self.node, dataO))
return (limitRemaining, self.cursor)
def send_data():
prog_lang = request.args['prog_lang']
code = request.args['code']
prog_input = request.args['prog_input']
print prog_lang
print code
print prog_input
proc_obj = Processor()
result = proc_obj.process(prog_lang, code, prog_input)
result = remove_non_ascii(result)
if not result:
result = "NO OUTPUT"
return render_template('output.html', result=result)
def __init__(self, config, shared):
Processor.__init__(self)
self.mtimes = {} # monitoring
self.shared = shared
self.config = config
self.up_to_date = False
self.watch_lock = threading.Lock()
self.watch_blocks = []
self.watch_headers = []
self.watched_addresses = {}
self.history_cache = {}
self.max_cache_size = 100000
self.chunk_cache = {}
self.cache_lock = threading.Lock()
self.headers_data = ""
self.headers_path = config.get("leveldb", "path")
self.mempool_values = {}
self.mempool_addresses = {}
self.mempool_hist = {} # addr -> (txid, delta)
self.mempool_hashes = set([])
self.mempool_lock = threading.Lock()
self.address_queue = Queue()
try:
self.test_reorgs = config.getboolean("leveldb", "test_reorgs") # simulate random blockchain reorgs
except:
self.test_reorgs = False
self.storage = Storage(config, shared, self.test_reorgs)
self.dblock = threading.Lock()
self.bitcoind_url = "http://%s:%s@%s:%s/" % (
config.get("bitcoind", "bitcoind_user"),
config.get("bitcoind", "bitcoind_password"),
config.get("bitcoind", "bitcoind_host"),
config.get("bitcoind", "bitcoind_port"),
)
self.sent_height = 0
self.sent_header = None
# catch_up headers
self.init_headers(self.storage.height)
self.blockchain_thread = threading.Thread(target=self.do_catch_up)
self.blockchain_thread.start()
def run(self):
"""执行函数"""
lang = os.path.join(PRJ_PATH, "source/")
reg_process = Processor(lang)
while True:
rect, name = self.queue.get()
start_time = time.time()
path = "c:/img/" + name + ".bmp"
width, height, _, _ = rect.split(',')
phone = reg_process.extract_phone(path, int(width), int(height))
print "phone: {0}; time: {1}".format(phone,
time.time() - start_time)
self.notice.put(
("NOTICE", EVENT.REG_PHONE, (name.split(",")[0], phone)))
class Worker(object):
def __init__(self, config, context, session):
self.config = config
self.context = context
self.session = session
self.receive_queue = self.config['queue_url']
self.cache = None
if self.config.get('cache'):
self.cache = memcache.Client([self.config['cache']])
self.processor = Processor(self.config, session, self.cache)
def run(self):
self.ses = self.session.client('ses')
log.info("Queue poll loop")
while True:
sqs = self.session.client('sqs')
messages = MessageIterator(sqs, self.receive_queue)
messages.msg_attributes = ['mtype', 'recipient']
for m in messages:
log.debug(
"Message id: %s received %s" % (
m['MessageId'], m.get('MessageAttributes', '')))
msg_kind = m.get('MessageAttributes', {}).get('mtype')
if msg_kind:
msg_kind = msg_kind['StringValue']
if msg_kind == DATA_MESSAGE:
self.processor.process_data_message(m)
elif msg_kind == EMAIL_MESSAGE:
try:
self.ses.send_raw_email(RawMessage=m['Body'])
except Exception as e:
log.exception("Unable to send raw message")
else:
log.warning("Unknown message format %s" % (m['Body'][:50]))
log.info("Processed Message")
messages.ack(m)
# Save 120s for actually sending out emails if we're batching.
# also want a clean exist at boundary instead of retrying
# sqs message.
remaining = self.context.get_remaining_time_in_millis()
if remaining < 120 * 1000:
self.processor.flush()
log.info("Exiting deadline")
return
log.info("Loop Complete / no messages")
return
def image_pose(image_path="exercise_images/look_up_right.png"):
args = cli()
img = cv2.imread(image_path)
height, width = img.shape[:2]
width_height = (
int(width * args.resolution // 16) * 16,
int(height * args.resolution // 16) * 16,
)
processor_singleton = Processor(width_height, args)
keypoint_sets, scores, width_height = processor_singleton.single_image(
b64image=base64.b64encode(cv2.imencode(".jpg", img)[1]).decode(
"UTF-8"))
print("keypoint:::", [keypoint_sets[0].tolist()])
# Use hard coding keypoints to visualise or adjust the keypoints
if args.override:
keypoint_sets = [[
[4.7093248e-01, 2.0426853e-01, 8.6702675e-01],
[4.8350826e-01, 1.9285889e-01, 9.0909678e-01],
[4.5933738e-01, 1.9329509e-01, 7.9952747e-01],
[5.0037909e-01, 2.1018410e-01, 6.8106771e-01],
[4.4295478e-01, 2.0732497e-01, 8.4605628e-01],
[5.1420957e-01, 2.8525087e-01, 9.5134568e-01],
[4.2685834e-01, 2.8133684e-01, 1.2184765e00],
[5.2738458e-01, 1.5919007e-01, 8.1505233e-01],
[4.1852579e-01, 1.7545088e-01, 7.9655707e-01],
[4.8712268e-01, 8.4518030e-02, 7.3338878e-01],
[4.5714667e-01, 8.1567958e-02, 1.0000000e-03],
[5.0157762e-01, 5.1679271e-01, 5.8042961e-01],
[4.4701675e-01, 4.9484652e-01, 7.6062948e-01],
[6.2654567e-01, 6.2967056e-01, 4.0503103e-01],
[4.7448894e-01, 7.3578298e-01, 3.6731312e-01],
[4.9454674e-01, 7.3082601e-01, 1.0000000e-03],
[4.9181211e-01, 8.8942599e-01, 4.6938220e-01],
]]
img = visualise(
img=img,
keypoint_sets=keypoint_sets,
width=width,
height=height,
vis_keypoints=True,
vis_skeleton=True,
)
cv2.imshow("Image Keypoints", img)
cv2.waitKey(0)
cv2.destroyAllWindows()
def test(dir):
final_rewards = []
for _ in range(5):
agent1 = GMSimAgent(log=False, dir=dir)
agent2 = GMSimAgent(log=False, dir=dir)
proc = Processor()
rewards = []
for i in range(1200):
r = proc.get_reward(agent1.next_step(), agent2.next_step())
rewards.append(r)
final_rewards.append(np.mean(rewards))
print("Test Reward %f with std %f" %
(np.mean(final_rewards), np.std(final_rewards)))
return np.mean(final_rewards)
def __init__(self, config, shared):
Processor.__init__(self)
self.mtimes = {} # monitoring
self.shared = shared
self.config = config
self.up_to_date = False
self.watch_lock = threading.Lock()
self.watch_blocks = []
self.watch_headers = []
self.watched_addresses = {}
self.history_cache = {}
self.max_cache_size = 100000
self.chunk_cache = {}
self.cache_lock = threading.Lock()
self.headers_data = ''
self.headers_path = config.get('leveldb', 'path')
self.mempool_values = {}
self.mempool_addresses = {}
self.mempool_hist = {}
self.mempool_hashes = set([])
self.mempool_lock = threading.Lock()
self.address_queue = Queue()
try:
self.test_reorgs = config.getboolean('leveldb', 'test_reorgs') # simulate random blockchain reorgs
except:
self.test_reorgs = False
self.storage = Storage(config, shared, self.test_reorgs)
self.dblock = threading.Lock()
self.darkcoind_url = 'http://%s:%s@%s:%s/' % (
config.get('darkcoind', 'darkcoind_user'),
config.get('darkcoind', 'darkcoind_password'),
config.get('darkcoind', 'darkcoind_host'),
config.get('darkcoind', 'darkcoind_port'))
self.sent_height = 0
self.sent_header = None
# catch_up headers
self.init_headers(self.storage.height)
self.blockchain_thread = threading.Thread(target = self.do_catch_up)
self.blockchain_thread.start()
class Worker(object):
def __init__(self, config, context, session):
self.config = config
self.context = context
self.session = session
self.receive_queue = self.config['queue_url']
self.cache = None
if self.config.get('cache'):
self.cache = memcache.Client([self.config['cache']])
self.processor = Processor(self.config, session, self.cache)
def run(self):
self.ses = self.session.client(
'ses', region_name=self.config.get('ses_region'))
log.info("Queue poll loop")
while True:
sqs = self.session.client('sqs')
messages = MessageIterator(sqs, self.receive_queue)
messages.msg_attributes = ['mtype', 'recipient']
for m in messages:
log.debug("Message id: %s received %s" %
(m['MessageId'], m.get('MessageAttributes', '')))
msg_kind = m.get('MessageAttributes', {}).get('mtype')
if msg_kind:
msg_kind = msg_kind['StringValue']
if msg_kind == DATA_MESSAGE:
self.processor.process_data_message(m)
elif msg_kind == EMAIL_MESSAGE:
try:
self.ses.send_raw_email(RawMessage=m['Body'])
except Exception as e:
log.exception("Unable to send raw message")
else:
log.warning("Unknown message format %s" % (m['Body'][:50]))
log.info("Processed Message")
messages.ack(m)
# Save 120s for actually sending out emails if we're batching.
# also want a clean exist at boundary instead of retrying
# sqs message.
remaining = self.context.get_remaining_time_in_millis()
if remaining < 120 * 1000:
self.processor.flush()
log.info("Exiting deadline")
return
log.info("Loop Complete / no messages")
return
def execute_processor(config_path, year):
config = Config(config_path)
processor = Processor(config)
processor.load_master()
if (year==''):
processor.process_data()
else:
processor.process_year(year)
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--filename", help="filename to process.")
args = parser.parse_args()
FILENAME = args.filename
with open(os.path.join('original_scripts', FILENAME + '.srt'), 'r') as f:
string = f.read()
tokenizer = Tokenizer()
tokens = tokenizer.tokenize(string)
parser = Parser()
script = parser.take_script(tokens)
processor = Processor()
processor.write_sentences(script, os.path.join('processed_scripts', FILENAME+'.txt'))
def poll(self):
events = {}
for monitor in self.monitors:
results = self._poll_with_retry(monitor)
logging.debug('{} events fetched')
for result in results:
if result.type not in self.event_types:
logging.debug(
'Found invalid event type: Event ID {} with type {}'.
format(result.id, result.type))
continue
events[result.id] = result
logging.debug('Found PushEvent {}'.format(result.id))
for _, event in events.items():
Processor.process_event(event)
def stop_recording(self, data_flag, stolen_yn):
self.record_flag = False
self.pre_tracking_time = 0
self.total_tracking_time = 0
# 1) Stop recording
self.camera.stop_recording()
# 2) Handling data and video in an extra thread
data_processor = Processor(self.file_name, data_flag, stolen_yn,
self.image_recognition_endpoint,
self.first_frame, self.pib_yn)
data_processor.start()
self.first_frame = ""
self.pib_yn = "n"
def main(self):
print "project root: " + PROJECT_ROOT
self.make_work_dir()
processor = Processor()
target_file_paths = self.build_targets()
for full_path in target_file_paths:
html = processor.process_html(full_path)
processed_file_name = full_path.replace(RESOURCE_HTML_DIR + "/", "").replace("/", "_")
processed_file_path = os.path.join(TARGET_DIR, processed_file_name)
print "Processed file: " + processed_file_path
with open(processed_file_path, 'w') as f:
f.write(html.encode("utf-8"))
def doWork(self):
self.processor = Processor()
self.processor.setdest(self.dest)
self.resopt = self.processor.resopt
items = self.resopt.get_all_unready()
n = 0
for item in items:
self.processor.process_dest_file(item)
n = n + 1
# 发送正在生成目标路径的信号
self.update.emit(n, item.fullpath)
if n % 100 == 0:
self.resopt.commit()
self.resopt.commit()
self.done.emit()
def __init__(self):
#初始化，驱动配置，延时等待配置，MongoDB配置
self.desired_caps = {
'platformName': PLATFORM,
'deviceName': DEVICE_NAME,
'appPackage': APP_PACKAGE,
'appActivity': APP_ACTIVITY
}
self.driver = webdriver.Remote(DRIVER_SERVER, self.desired_caps)
self.wait = WebDriverWait(self.driver, TIMEOUT)
self.client = MongoClient(MONGO_URL)
self.db = self.client[MONGO_DB]
self.collection = self.db[MONGO_COLLECTION]
#处理时间，先要创建一个对象
self.processor = Processor()
def run_bot(program: List[int], part2: bool = False) -> Dict[Point, int]:
running_bot = Processor(program).run_on_output_generator(output_batch=2)
next(running_bot) # Prime the pump to the first yield for .send( below
location = Point(0, 0)
hull = {} if not part2 else {location: 1}
direction = "^"
try:
while True:
color = hull.get(location, 0)
new_color, turn = running_bot.send(color)
hull[location] = new_color
direction = TURN_DB[direction][turn]
location += DIR_VEC[direction]
except StopIteration:
return hull
def test_creates_table(self):
columns = (("biba", "int"), ("baba", "str"), ("buba", "datetime"))
name = "database"
proc = Processor(self.conn, name, columns, [])
proc.create()
with self.conn.cursor() as c:
sql = """
SELECT column_name, udt_name
FROM INFORMATION_SCHEMA.COLUMNS
WHERE table_name = '%s'
""" % (name, )
c.execute(sql)
resp_columns = c.fetchall()
self.assertEqual([(c[0], TYPE_MAP[c[1]].lower()) for c in columns],
resp_columns)
class Simulator(): def __init__(self, args): self.args = args self.memory = Memory() if self.args.exeUnits: config.NUMBER_EXECUTION_UNITS = int(args.exeUnits) if self.args.step: print "Executing " + str(self.args.file) + " with " + str(config.NUMBER_EXECUTION_UNITS) + " execution units" def run(self): self.assembledProgram = Assembler(self.memory, args) self.processor = Processor(self.memory, args) self.processor.run(self.assembledProgram)
def start_button(self):
if self.textbox_isEmpty(user_input) and self.textbox_isEmpty(regex_input):
showinfo("No input given", "Please write something in one of the input boxes")
elif self.entry_isEmpty(filename):
showinfo("No file given", "Please select a file to continue")
elif self.entry_isEmpty(search_col):
showinfo("Search column empty", "Please write a value in search box")
elif self.entry_isEmpty(result_col):
showinfo("Result column empty", "Please write a value in result box")
elif not self.entry_isEmpty(filename) and not os.path.isfile(str(filename.get())):
showwarning("File does not exist", "Please select a valid file")
else:
processor = Processor(str(filename.get()), user_input.get('1.0', 'end-1c'), regex_input.get('1.0', 'end-1c'))
processor.process_rows(search_col=int(search_col.get()), result_col=int(result_col.get()))
finish_label.config(text="Finished Processing")
def __init__(self, processor=None):
if (processor == None):
self.processor = Processor()
else:
self.processor = processor
self.actions.SetProcessor(self.processor)
def __app(self, app_idx, comms_per_app, event_submitted):
data_length = 4 * 1e3 # 4 MB data size
for idx in range(
self.__cryptogen.randrange(comms_per_app[1] -
comms_per_app[0]) +
comms_per_app[0]):
queue_idx = self.__cryptogen.randrange(self.__processor.num_cores)
comm_idx = "{:d}-{:d}-{:d}".format(app_idx, idx, queue_idx)
# generate the workload at localhost
event = yield self.__env.process(
self.__processor.submit(
Processor.Task(
comm_idx + "-GENERATE", queue_idx,
data_length * self._CPU_CYCLES_PER_KILOBYTE_GENERATE)))
yield event
# receive the workload by remote hosts
# yield self.__env.timeout(data_length *
# self._RECEIVE_SUBMISSION_COMM_DELAY_PER_KILOBYTE)
submit_comm = SubmissionCommand(comm_idx, data_length, True, 0.5,
self.__env.now)
yield self.submission_queues[queue_idx].put(submit_comm)
self.summary["num_commands_submitted"] += 1
self.summary["total_data_bytes"] += data_length
event_submitted.succeed()
def run_bot(program: List[int]) -> Dict[Point, int]:
bot = Processor(program)
points_to_scan = [Point(y, x) for x in range(50) for y in range(50)]
scan = {}
for point in points_to_scan:
scan[point] = check_location(bot, program, point)
return scan
def main():
parser = argparse.ArgumentParser(description='Part-of-Speech Tagging.')
parser.add_argument(
'--prefix',
'-p',
type=str,
default='',
help='specify prefix of files which will be used to store model')
parser.add_argument('--times',
'-t',
type=int,
default=1,
help='specify iteration times')
parser.add_argument(
'--all',
'-a',
action='store_true',
help='without this switch, model will be trained by random sampled data'
)
parser.add_argument('--file',
'-f',
type=str,
default='',
help='specify test data file')
parser.add_argument('--save',
'-s',
action='store_true',
help='enable this to save model file')
args = parser.parse_args()
tagger = Tagger('data/wsj00-18.pos', args.times, not args.all, args.save,
args.prefix)
test_data = Processor(args.file)
tagger.benchmark(test_data)
def part2():
progstring = load("7.txt")[0]
program = [int(inst) for inst in progstring.split(",")]
settings = list(permutations(
[9, 8, 7, 6, 5], 5)) # all possible permutations of the phase settings
max = 0
for setting in settings:
processors = [Processor(program) for n in range(5)]
# send phase settings to each
for i in range(5):
out = processors[i].run([setting[i]])
# now start them all running
lastE = None
output = 0
while sum([p.runState for p in processors
]) > 0: # The runState is zero when the processor is ended
for i in range(5):
output = processors[i].run(
[output]
) # pass the output of this processor as the input of the next
# The processors now pause when they run out of input, remembering their current line position
if output is not None:
lastE = output
if lastE > max:
max = lastE
print(max)
def quant_boro_species(boro):
try:
json = fetcher.check_cache()
data = Processor.count_by_boro(data=json, boro=boro)
return {'data': data}
except:
abort(400, 'Bad request, check your query string parameters')
def __init__(self, config, shared):
Processor.__init__(self)
self.daemon = True
self.config = config
self.shared = shared
self.irc_queue = Queue.Queue()
self.peers = {}
if self.config.get('server', 'irc') == 'yes':
self.irc = IrcThread(self, self.config)
self.irc.start(self.irc_queue)
t = threading.Thread(target=self.read_irc_results)
t.daemon = True
t.start()
else:
self.irc = None
def test_add_with_existing_card_number(self):
'''
This test will fail because I have yet to decide if the summary
should print error or the current balance
'''
self.processor = Processor({
"Tom": [{
"card_number": "4111111111111111",
"balance": 0,
"limit": 1000
}]
})
commands = [["Add", "Tom", "1231231321", "$1000"]]
self.processor.process(commands)
self.assertEqual({"Tom": "error"}, self.processor.summary)
def __process_command(self, queue_idx):
while True:
with self.completion_queues[queue_idx].get() as slot:
comm = yield slot
data_packet = self.read_cache(comm.id)
if comm.req_isp and not comm.ret_isp:
event = yield self.__env.process(
self.__processor.submit(
Processor.Task(
comm.id + "-COMPUTE", queue_idx,
data_packet.length *
self._CPU_CYCLES_PER_KILOBYTE_COMPUTE,
Processor.Task.PRIORITY_HIGH)))
core_id = yield event
assert core_id == queue_idx # nosec
self.__logger.bind(by_core=core_id).trace(comm)
self.summary["num_commands_completed"] += 1
if (self.summary["num_commands_submitted"]
== self.summary["num_commands_completed"]
and self.__all_command_submitted.processed):
self.__shutdown_hook.succeed()
def run_migration(self):
if not hasattr(self,'processor'):
self.processor = Processor(self.model)
print "running migration"
def hello_world():
global prcsr
if prcsr is None:
prcsr = Processor()
if not os.path.exists(UPLOAD_FOLDER):
os.makedirs("./uploads")
return render_template('base.html')
def test_jira_id_extraction(self):
slack_client = MagicMock()
processor = Processor("target", ["prefix1-", "something2_", "bug-"], slack_client)
tests = [
("prefix1-not-jira-issue", None),
("prefix1-some-text-and-a-number-9999-like-this", None), # Jira id has to be immediately after prefix
("prefix1-number-9999-like-this", "number-9999"),
("bug-xyz-1234", "xyz-1234"),
("bug-X1-01234", "X1-01234"),
("bug-X-1", None), # Project prefix has two have at least two characters
("prefix1-ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-1234", None), # Just..no
("prefix1-IM-1234-some-description", "IM-1234"),
("something2_IM-1234-some-description", "IM-1234"),
]
for (name, jira_id) in tests:
self.assertEqual(jira_id, processor._extract_jira_id(name))
async def core():
process_cmd_arguments()
prepare_loggers()
if test_collector_path is not None:
return await test_collector.run(test_collector_path)
proxy_processor = Processor.get_instance()
try:
code = await asyncio.gather(*[
proxy_processor.worker(),
statistics.worker(),
materialized_view_updater.worker(),
])
BaseChecker.clean()
return code
except KeyboardInterrupt:
pass
except BaseException as ex:
main_logger.exception(ex)
print("critical error happened, see logs/main.log")
return 1
return 0
def count_per_nta():
try:
json = fetcher.check_cache()
data = Processor.count_per_nta(data=json)
return {'data': data}
except:
abort(500, 'Something went wrong on our end :(')
def test_getLanguageProbabilities_from_word_returns_list_of_tuples_when_data_available(
self):
self.assertTrue(
isinstance(
Processor.getLanguageProbabilitiesFromWord(
self.getPreparedFakeProcessor(), self.test_word)[0],
tuple))
def test_download_single(self):
with Processor() as p:
# The endpoint returns "Hello, world!".
path = p._download_single(self.url + '/download/test.txt')
self.assertTrue(path.endswith('.txt'))
with open(path, 'rb') as f:
self.assertEqual(f.read(), b'Hello, world!')
def __init__(self, config, shared):
Processor.__init__(self)
self.daemon = True
self.config = config
self.shared = shared
self.irc_queue = Queue.Queue()
self.peers = {}
if self.config.get('server', 'irc') == 'yes':
self.irc = IrcThread(self, self.config)
self.irc.start(self.irc_queue)
t = threading.Thread(target=self.read_irc_results)
t.daemon = True
t.start()
else:
self.irc = None
