def test_ddGraph(self):
"""
Graph is using dd to read a file and write to another. This is mainly
to test that the separatorString parameter is working correctly.
"""
sessionId = "lalo"
ddGraph = "graphs/ddTest.graph"
with pkg_resources.resource_stream("test",
ddGraph) as f: # @UndefinedVariable
logger.debug(f"Loading graph: {f}")
graphSpec = json.load(f)
self.createSessionAndAddGraph(sessionId, graphSpec=graphSpec)
# Deploy now and get OIDs
bs = graphSpec[0]["applicationArgs"]["bs"]["value"]
count = graphSpec[0]["applicationArgs"]["count"]["value"]
self.dim.deploySession(sessionId)
a, c = [
self.dm._sessions[sessionId].drops[x]
for x in ("2022-02-11T08:05:47_-5_0", "2022-02-11T08:05:47_-3_0")
]
data = os.urandom(bs * count)
logger.debug(f"Length of data produced: {len(data)}")
with droputils.DROPWaiterCtx(self, c, 3):
a.write(data)
a.setCompleted()
self.assertEqual(data, droputils.allDropContents(c))
def test_namedPorts(self):
"""
Use a graph with named ports and check whether it is runnning
"""
init_oid = "2022-03-20T04:33:27_-2_0" # first drop in graph
sessionId = "lalo"
with pkg_resources.resource_stream(
"test", "graphs/funcTestPG_namedPorts.graph"
) as f: # @UndefinedVariable
graphSpec = json.load(f)
# dropSpecs = graph_loader.loadDropSpecs(graphSpec)
self.createSessionAndAddGraph(sessionId, graphSpec=graphSpec)
# Deploy now and get OIDs
self.dim.deploySession(sessionId)
fd = self.dm._sessions[sessionId].drops["2022-03-20T04:33:27_-1_0"]
init_drop = self.dm._sessions[sessionId].drops[init_oid]
a = InMemoryDROP("a", "a")
init_drop.addInput(a)
logger.debug(f"PyfuncAPPDrop: {dir(fd)}")
for i in fd.parameters["inputs"]:
logger.debug(f"PyfuncAPPDrop input names:{i}")
with droputils.DROPWaiterCtx(self, init_drop, 3):
a.setCompleted()
def DeleteNoLocal(self, _nodellist):
try:
if not _nodellist and len(_nodellist) <= 0:
return
module_path = os.path.join(self.ApacheDocsPath, "modules")
if os.path.exists(module_path):
m_paths = getListFiles(module_path)
for path in m_paths:
if not (path.lower() in _nodellist):
os.remove(path)
logger.debug("删除文件", path)
if os.path.exists(module_path):
m_dirs = getListDirs(module_path)
pathstr = "C:\\thunder\\Apache\\htdocs\\modules\\moduledatatype"
for dir in m_dirs:
print(dir)
if pathstr.lower() == dir.lower():
continue
if not (dir in _nodellist):
print(os.path.getmtime(dir))
mtime = time.strftime(
'%m', time.localtime(os.path.getmtime(dir)))
ctime = time.strftime('%m',
time.localtime(time.time()))
if os.path.exists(dir) and (int(ctime) - int(mtime)
>= 1):
try:
os.remove(dir)
except:
pass
except Exception as e:
print("traceback.format_exc()", traceback.format_exc())
def test_namedPorts_with_kwonlyargs(self):
"""
Use a graph with named ports and check whether it is runnning
"""
init_oids = [
"2022-03-30T03:46:01_-2_0",
"2022-03-30T03:46:01_-6_0",
] # first drops in graph
sessionId = "lalo"
with pkg_resources.resource_stream(
"test",
"graphs/pyfunc_glob_testPG.graph") as f: # @UndefinedVariable
graphSpec = json.load(f)
# dropSpecs = graph_loader.loadDropSpecs(graphSpec)
self.createSessionAndAddGraph(sessionId, graphSpec=graphSpec)
# Deploy now and get OIDs
self.dim.deploySession(sessionId)
fd = self.dm._sessions[sessionId].drops["2022-03-30T03:46:01_-1_0"]
i = 0
start_drops = [InMemoryDROP(x, x) for x in ("a", "b")]
for oid in init_oids:
init_drop = self.dm._sessions[sessionId].drops[oid]
init_drop.addInput(start_drops[i])
i += 1
logger.debug(f"PyfuncAPPDrop: {dir(fd)}")
for i in fd.parameters["inputs"]:
logger.debug(f"PyfuncAPPDrop input names:{i}")
with droputils.DROPWaiterCtx(self, init_drop, 3):
[a.setCompleted() for a in start_drops]
def nearest_ratings(user_id, count, address):
"""Get Nearest Recommended business to user's address """
logger.debug("User %s rating requested for nearest address", user_id,
address)
ratings = recommendationengine.get_nearest_businesses(
user_id, count, address)
return json.dumps(ratings)
def _do_waitpid(self, loop, expected_pid, callback, args):
assert expected_pid > 0
try:
pid, status = os.waitpid(expected_pid, 0)
except ChildProcessError:
# The child process is already reaped
# (may happen if waitpid() is called elsewhere).
pid = expected_pid
returncode = 255
logger.warning(
"Unknown child process pid %d, will report returncode 255", pid
)
else:
returncode = _compute_returncode(status)
if loop.get_debug():
logger.debug(
"process %s exited with returncode %s", expected_pid, returncode
)
if loop.is_closed():
logger.warning("Loop %r that handles pid %r is closed", loop, pid)
else:
loop.call_soon_threadsafe(callback, pid, returncode, *args)
self._threads.pop(expected_pid)
def _on_handshake_complete(self, handshake_exc):
self._in_handshake = False
sslobj = self._sslpipe.ssl_object
try:
if handshake_exc is not None:
raise handshake_exc
peercert = sslobj.getpeercert()
if not hasattr(self._sslcontext, "check_hostname"):
# Verify hostname if requested, Python 3.4+ uses check_hostname
# and checks the hostname in do_handshake()
if (self._server_hostname
and self._sslcontext.verify_mode != ssl.CERT_NONE):
ssl.match_hostname(peercert, self._server_hostname)
except BaseException as exc:
if self._loop.get_debug():
if isinstance(exc, ssl.CertificateError):
logger.warning(
"%r: SSL handshake failed "
"on verifying the certificate",
self,
exc_info=True,
)
else:
logger.warning("%r: SSL handshake failed",
self,
exc_info=True)
self._transport.close()
if isinstance(exc, Exception):
self._wakeup_waiter(exc)
return
else:
raise
if self._loop.get_debug():
dt = self._loop.time() - self._handshake_start_time
logger.debug("%r: SSL handshake took %.1f ms", self, dt * 1e3)
# Add extra info that becomes available after handshake.
self._extra.update(
peercert=peercert,
cipher=sslobj.cipher(),
compression=sslobj.compression(),
ssl_object=sslobj,
)
if self._call_connection_made:
self._app_protocol.connection_made(self._app_transport)
self._wakeup_waiter()
self._session_established = True
# In case transport.write() was already called. Don't call
# immediately _process_write_backlog(), but schedule it:
# _on_handshake_complete() can be called indirectly from
# _process_write_backlog(), and _process_write_backlog() is not
# reentrant.
self._loop.call_soon(self._process_write_backlog)
def _start_handshake(self):
if self._loop.get_debug():
logger.debug("%r starts SSL handshake", self)
self._handshake_start_time = self._loop.time()
else:
self._handshake_start_time = None
self._in_handshake = True
# (b'', 1) is a special value in _process_write_backlog() to do
# the SSL handshake
self._write_backlog.append((b"", 1))
self._loop.call_soon(self._process_write_backlog)
def resume_writing(self):
assert self._paused
self._paused = False
if self._loop.get_debug():
logger.debug("%r resumes writing", self)
waiter = self._drain_waiter
if waiter is not None:
self._drain_waiter = None
if not waiter.done():
waiter.set_result(None)
def resume_writing(self):
assert self._paused
self._paused = False
if self._loop.get_debug():
logger.debug("%r resumes writing", self)
waiter = self._drain_waiter
if waiter is not None:
self._drain_waiter = None
if not waiter.done():
waiter.set_result(None)
def parse_query(self, parser):
show, season, episode, title = parser.parse()
show = os.path.basename(show)
if show.islower():
show = show.title()
self.context['Name'], self.context['TV Season'] = title, season
for tag in ['Artist', 'Album Artist', 'TV Show']:
self.context[tag] = show
self.context['TV Episode #'] = self.context['Track #'] = episode
self.context['TV Episode ID'] = 'S{}E{}'.format(season, episode)
self.context['Album'] = '{}, Season {}'.format(show, season)
LOGGER.debug('Trakt Query: %s', show)
return show
def _fatal_error(self, exc, message="Fatal error on transport"):
# Should be called from exception handler only.
if isinstance(exc, base_events._FATAL_ERROR_IGNORE):
if self._loop.get_debug():
logger.debug("%r: %s", self, message, exc_info=True)
else:
self._loop.call_exception_handler({
"message": message,
"exception": exc,
"transport": self._transport,
"protocol": self,
})
if self._transport:
self._transport._force_close(exc)
def _save_status_in_file(self, file_name: str = ''):
"""Print in file the status history."""
if file_name == '':
checkFolders(RESULTS_FOLDER)
file_name = f'{RESULTS_FOLDER}/status_history_{"".join(self.name_.split())}.st'
logger.debug(
f'Storing status from node {self.name_} in file {file_name}')
with open(f'{file_name}', 'w') as file:
for time, status in self.dds_status_node_.status_history_:
file.write(
f'TIME: {time}\n{AmlDdsStatusNode._str_status_data(status)}\n'
)
def _read_ready(self):
message = self._pubsub.get_message()
if message:
if isinstance(message['data'], bytes):
self._protocol.data_received(message['data'])
else:
if self._loop.get_debug():
logger.debug("%r received EOF", self)
keep_open = self._protocol.eof_received()
if keep_open:
# We're keeping the connection open so the
# protocol can write more, but we still can't
# receive more, so remove the reader callback.
self._loop.remove_reader(self._sock_fd)
else:
self.close()
def _get_pi_solution(split_graph):
"""
1. create H (admissable graph) based on Section 3
http://fmdb.cs.ucla.edu/Treports/930014.pdf
2. calculate the max flow f' on H using networkx
3. construct Residual graph from f' on H based on
https://www.topcoder.com/community/data-science/\
data-science-tutorials/minimum-cost-flow-part-two-algorithms/
4. calculate Pi based on Section 3 again
"""
# Step 1
H = nx.DiGraph()
H.add_nodes_from(split_graph)
for ed in split_graph.edges(data=True):
Cxy = int(ed[2].get("capacity", sys.maxsize))
Axy = int(ed[2]["weight"])
logger.debug(f"Found capacity and weight: {Axy}, {Cxy}")
if Axy == 0 and Cxy > 0:
H.add_edge(ed[0], ed[1], capacity=Cxy, weight=Axy)
# Step 2
flow_value, flow_dict = nx.maximum_flow(H, "s", "t")
# Step 3
R = nx.DiGraph()
R.add_nodes_from(H)
for ed in H.edges(data=True):
Xij = flow_dict[ed[0]][ed[1]]
Uij = ed[2].get("capacity", sys.maxsize)
Cij = ed[2]["weight"]
if (Uij - Xij) > 0:
R.add_edge(ed[0], ed[1], weight=Cij)
if Xij > 0:
R.add_edge(ed[1], ed[0], weight=-1 * Cij)
# Step 4
pai = dict()
for n in R.nodes():
if nx.has_path(R, "s", n):
pai[n] = 0
else:
pai[n] = 1
return pai
def eof_received(self):
"""Called when the other end of the low-level stream
is half-closed.
If this returns a false value (including None), the transport
will close itself. If it returns a true value, closing the
transport is up to the protocol.
"""
try:
if self._loop.get_debug():
logger.debug("%r received EOF", self)
self._wakeup_waiter(ConnectionResetError)
if not self._in_handshake:
keep_open = self._app_protocol.eof_received()
if keep_open:
logger.warning("returning true from eof_received() "
"has no effect when using ssl")
finally:
self._transport.close()
def has_artwork(self, url):
"""Attempts to download artwork from the provided URL and write it to a
.jpg file named '.albumart.jpg' then return True as long as a 2xx HTTP
response is recieved. If an error should occur, nothing is downloaded
and False is returned
"""
if not url:
return False
LOGGER.info('Downloading Album Artwork...')
LOGGER.debug('URL: %s', url)
req = requests.get(url)
if 200 <= req.status_code < 300:
file_name = '.albumart{}.jpg'.format(str(uuid4()))
LOGGER.info('Writing artwork to %s', file_name)
with open(file_name, 'wb') as f:
f.write(req.content)
return file_name.replace(' ', '\\ ')
message = 'Album Art Not Downloaded: {}'.format(req.status_code)
LOGGER.warn(message)
return False
def DeleteVLocalModule(self, list):
for item in list:
fileName = os.path.basename(item.fileurl)
fileName = os.path.join(self.ApacheDocsPath, "modules", fileName)
#权限设置
if os.stat(fileName).st_mode == os.stat.S_IREAD:
os.chmod(os.stat.S_IRWXU)
if os.path.exists(fileName):
os.remove(fileName)
logger.debug("删除文件", fileName)
#TODO 删除
bll = KtvModuleVerBll()
bll.DeleteVer(item.id)
dir = item.uppath
mtime = time.strftime('%m', os.path.getmtime(dir))
ctime = time.strftime('%m', time.localtime(time.time()))
if os.path.exists(dir) and (int(ctime) - int(mtime) >= 1):
os.remove(dir)
def test_pos_only_args(self):
"""
Use a graph with compile function to test positional only arguments
"""
sessionId = "lalo"
with pkg_resources.resource_stream(
"test", "graphs/compilePG.graph") as f: # @UndefinedVariable
graphSpec = json.load(f)
# dropSpecs = graph_loader.loadDropSpecs(graphSpec)
self.createSessionAndAddGraph(sessionId, graphSpec=graphSpec)
# Deploy now and get OIDs
self.dim.deploySession(sessionId)
sd = self.dm._sessions[sessionId].drops["2022-05-06T08:43:26_-2_0"]
fd = self.dm._sessions[sessionId].drops["2022-05-06T08:43:26_-1_0"]
with droputils.DROPWaiterCtx(self, fd, 3):
sd.setCompleted()
#logger.debug(f'PyfuncAPPDrop signature: {dir(fd)}')
logger.debug(f'PyfuncAPPDrop status: {fd.status}')
self.assertEqual(2, fd.status)
def test_simple(self):
"""This creates the following graph
1 --|
|-- add --> the_sum --------|
2 --| | |--> part1 --|
|--> divide --> division -->partition --| |--> add -> final
4 --| | |--> part2 --|
|-- substract --> the_sub --|
3 --|
"""
delayed = self.delayed
compute = self.compute
the_sum = delayed(add)(1.0, 2.0)
the_sub = delayed(subtract)(4.0, 3.0)
division = delayed(divide)(the_sum, the_sub)
parts = delayed(partition, nout=2)(division)
logger.debug(f"partitions: {type(parts)}")
result = 3.0
result = compute(delayed(add)(*parts))
self.assertEqual(3.0, result)
def test_ArrayLoop(self):
"""
Use a graph with compile function to test positional only arguments
"""
sessionId = "lalo"
start_drop = InMemoryDROP('a', 'a')
with pkg_resources.resource_stream(
"test", "graphs/ArrayLoopPG.graph") as f: # @UndefinedVariable
graphSpec = json.load(f)
# dropSpecs = graph_loader.loadDropSpecs(graphSpec)
self.createSessionAndAddGraph(sessionId, graphSpec=graphSpec)
# Deploy now and get OIDs
self.dim.deploySession(sessionId)
sd = self.dm._sessions[sessionId].drops["2022-06-22T09:13:53_-1_0"]
sd.addInput(start_drop)
fd = self.dm._sessions[sessionId].drops["2022-06-22T09:13:53_-4_0/0/0"]
with droputils.DROPWaiterCtx(self, fd, 3):
start_drop.setCompleted()
#logger.debug(f'PyfuncAPPDrop signature: {dir(fd)}')
logger.debug(f'PyfuncAPPDrop status: {fd.status}')
self.assertEqual(2, fd.status)
def tearDownClass(cls):
logger.debug(f"Removing temp directory {cls._temp}")
shutil.rmtree(cls._temp, True)
def get_user_Ids():
logger.debug("Users")
userIds = recommendationengine.get_User_Ids()
return json.dumps(userIds)
def mainIndex():
logger.debug("User %s rating requested for movie %s")
return render_template('index.html')
def business_ratings(user_id, business_id):
logger.debug("User %s rating requested for business %s", user_id,
business_id)
ratings = recommendationengine.get_ratings_for_business_ids(
user_id, [business_id])
return json.dumps(ratings)
def ratings_within_category(user_id, count, category):
"""Get Top Recommendations within the category"""
logger.debug("User %s rating requested for category %s", user_id, category)
ratings = recommendationengine.get_business_in_categories(
user_id, count, category)
return json.dumps(ratings)
def ratings_within_state(user_id, count, state):
"""Get Top Recommendations within the state """
logger.debug("User %s rating requested for state %s", user_id, state)
ratings = recommendationengine.get_business_in_state(user_id, count, state)
data = json.dumps(ratings)
return data
def pause_writing(self):
assert not self._paused
self._paused = True
if self._loop.get_debug():
logger.debug("%r pauses writing", self)
def test_fullRound(self):
"""
A test that exercises most of the REST interface exposed on top of the
DataIslandManager
"""
sessionId = "lala"
restPort = 8989 # don't interfere with EAGLE default port
args = ["--port", str(restPort), "-N", hostname, "-qqq"]
dimProcess = tool.start_process("dim", args)
with testutils.terminating(dimProcess, timeout=10):
# Wait until the REST server becomes alive
self.assertTrue(
utils.portIsOpen("localhost", restPort, timeout=10),
"REST server didn't come up in time",
)
# The DIM is still empty
sessions = testutils.get(self, "/sessions", restPort)
self.assertEqual(0, len(sessions))
dimStatus = testutils.get(self, "", restPort)
self.assertEqual(1, len(dimStatus["hosts"]))
self.assertEqual(hostname, dimStatus["hosts"][0])
self.assertEqual(0, len(dimStatus["sessionIds"]))
# Create a session and check it exists
testutils.post(
self, "/sessions", restPort, '{"sessionId":"%s"}' % (sessionId)
)
sessions = testutils.get(self, "/sessions", restPort)
self.assertEqual(1, len(sessions))
self.assertEqual(sessionId, sessions[0]["sessionId"])
self.assertDictEqual(
{hostname: SessionStates.PRISTINE}, sessions[0]["status"]
)
# Add this complex graph spec to the session
# The UID of the two leaf nodes of this complex.js graph are T and S
# Since the original complexGraph doesn't have node information
# we need to add it manually before submitting -- otherwise it will
# get rejected by the DIM.
with pkg_resources.resource_stream(
"test", "graphs/complex.js"
) as f: # @UndefinedVariable
complexGraphSpec = json.load(codecs.getreader("utf-8")(f))
logger.debug(f"Loaded graph: {f}")
for dropSpec in complexGraphSpec:
dropSpec["node"] = hostname
testutils.post(
self,
"/sessions/%s/graph/append" % (sessionId),
restPort,
json.dumps(complexGraphSpec),
)
self.assertEqual(
{hostname: SessionStates.BUILDING},
testutils.get(self, "/sessions/%s/status" % (sessionId), restPort),
)
# Now we deploy the graph...
testutils.post(
self,
"/sessions/%s/deploy" % (sessionId),
restPort,
"completed=SL_A,SL_B,SL_C,SL_D,SL_K",
mimeType="application/x-www-form-urlencoded",
)
self.assertEqual(
{hostname: SessionStates.RUNNING},
testutils.get(self, "/sessions/%s/status" % (sessionId), restPort),
)
# ...and write to all 5 root nodes that are listening in ports
# starting at 1111
msg = os.urandom(10)
for i in range(5):
utils.write_to(
"localhost", 1111 + i, msg, 2
), "Couldn't write data to localhost:%d" % (1111 + i)
# Wait until the graph has finished its execution. We'll know
# it finished by polling the status of the session
while (
SessionStates.RUNNING
in testutils.get(
self, "/sessions/%s/status" % (sessionId), restPort
).values()
):
time.sleep(0.2)
self.assertEqual(
{hostname: SessionStates.FINISHED},
testutils.get(self, "/sessions/%s/status" % (sessionId), restPort),
)
testutils.delete(self, "/sessions/%s" % (sessionId), restPort)
sessions = testutils.get(self, "/sessions", restPort)
self.assertEqual(0, len(sessions))
def pause_writing(self):
assert not self._paused
self._paused = True
if self._loop.get_debug():
logger.debug("%r pauses writing", self)
