def test_tiny_1():
requests = []
requests.append(Request("Stanford", 0, 1))
requests.append(Request("MIT", 2, 3))
requests.append(Request("UC Berkeley", 4, 5))
run_tests(requests, 6, "Tiny 1")
def compensationRequest(component, blood_type, units): #create request req = Request(component, blood_type, units, 2) #serve request req.flow() #check stock level checkStockLevel(component, blood_type)
def test_tiny_3():
requests = []
requests.append(Request("Stanford", 0, 2))
requests.append(Request("MIT", 1, 9))
requests.append(Request("UC Berkeley", 4, 5))
run_tests(requests, 9, "Tiny 3")
def Login(username, password):
path = "/"
url = Request.MakeURL(Host, path, {}, TLS)
(resp_header, resp_body) = Request.Request(url)
session_cookie = ""
for item in resp_header:
if item[0] == "Set-Cookie" and "session" in item[1]:
session_cookie = item[1].split(";")[0]
#print(session_cookie)
csrf_nonce = ""
match = re.findall("csrf_nonce = \"([0-9a-f]+?)\"",
resp_body.decode("utf-8"))
if len(match) > 0:
csrf_nonce = match[0]
#print(csrf_nonce)
path = "/login"
url = Request.MakeURL(Host, path, {}, TLS)
header = {
"Content-Type": "application/x-www-form-urlencoded",
"Cookie": session_cookie,
}
query = {
"name": username,
"password": password,
"nonce": csrf_nonce,
}
body = Request.MakeBody(query, header)
(resp_header, resp_body) = Request.Request(url, body, header)
for item in resp_header:
if item[0] == "Set-Cookie" and "session" in item[1]:
session_cookie = item[1].split(";")[0]
#print(session_cookie)
HEADER["cookie"] = session_cookie
return
def test_tiny_4():
requests = []
requests.append(Request("Stanford", 1, 3))
requests.append(Request("MIT", 4, 9))
requests.append(Request("UC Berkeley", 0, 7))
run_tests(requests, 9, "Tiny 4")
def test_tiny_2():
requests = []
requests.append(Request("Stanford", 0, 1))
requests.append(Request("MIT", 1, 2))
requests.append(Request("UC Berkeley", 2, 3))
run_tests(requests, 4, "Tiny 2")
def issueRequest(self):
if not self.active:
return
request = Request()
request.createdAt = self.sim.now
request.onCompleted = lambda: self.onCompleted(request)
#self.sim.log(self, "Requested {0}", request)
self.server.request(request)
def issueRequest(self):
if not self.active:
return
request = Request()
request.createdAt = self.sim.now
request.onCompleted = lambda: self.onCompleted(request)
#self.sim.log(self, "Requested {0}", request)
self.server.request(request)
def HandleClient(clientsock):
name = threading.currentThread().getName()
print name, ' Started.............'
global EOM
chunks=[]
while 1:
buf = clientsock.recv(2048)
chunks.append(str(buf))
if (EOM in chunks[-1]):
msg= "".join(chunks)[:-5]
if (msg=="TEST: HELLO"):
return
req =Request.parseRequest(msg)
if (not validReq(req)):
Response.sendData(clientsock,"Invalid Request")
print "invalid request"
clientsock.close()
return
if (req["requestType"]=="SUBMIT"):
job=Request.parseJob(msg)
global JOBS
if (req["Owner"]=="system" or req["Owner"]=="utils"):
res= PBS.run(job["command"],req["JobID"])
if req["Owner"]=="system":
Response.sendData(clientsock,"Done")
else:
Response.sendData(clientsock,res)
elif req["Owner"]=="ubuntu":
res= PBS.run("su ubuntu -c '"+job["command"]+"'",req["JobID"])
Response.sendData(clientsock,res)
elif req["Owner"]=="ehpcuser":
res= PBS.run("su ehpcuser -c '"+job["command"]+"'",req["JobID"])
Response.sendData(clientsock,res)
else:
print "command:" + job["command"]
# print "inputs:" + job["inputs"]
c = PBS.AddFileWrapper("direct",job["command"],job["inputs"],job["outputs"])
id= PBS.runAsPBSJob(req["Owner"],req["JobID"],c)
Response.sendData(clientsock,'recieved:'+id)
clientsock.close()
elif (req["requestType"]=="STATUS"):
status=Job.checkPBSstatus(req["JobID"])
Response.sendData(clientsock,Job.perpareJobStatusMessage(req,status))
elif (req["requestType"]=="FETCH"):
OutputList = Request.getOutputList(msg)
response = Response.generateResponse(msg,OutputList)
Response.sendData(clientsock,response)
elif (req["requestType"]=='UPLOAD'):
UploadDict = Request.parseUpload(msg)
for key, value in UploadFileData.iteritems():
createFile = open(key, "wb")
while True:
data = conn.recv(1024)
createFile.write(value)
createFile.close()
break
def compute_result_table():
# table_path = "/Users/khoihd/Downloads/MAP_test/g_stable_tree_fix_5/comparison_new_config"
# table_path = "/Users/khoihd/Downloads/MAP_test/g_stable_tree_fix_5/hardcode_comparison_new_config"
table_path = "/Users/khoihd/Downloads/MAP_test/MAP_timestamp/hardcode_comparison_new_config"
# table_path = "/Users/khoihd/Downloads/MAP_test/null_pointer_test_run/hardcode_comparison_new_config"
algorithms = ["RC-DIFF"]
agents = [5, 10, 15, 20]
# (topology, instances) = ("scale-free-tree", range(10))
(topology, instances) = ("random-network", range(1))
cutoff_timestep = 16
for agent in agents:
for alg in algorithms:
avg_request = Request.Request()
for instance in instances:
instance_path = table_path + "/" + alg + "/scenario/" + topology + "/d" + str(
agent) + "/" + str(instance)
print(instance_path)
unzip_output(instance_path, alg)
_, region_result, request_total, over_time_result = getRequestsOverTime(
alg, instance_path)
cutoff_success, cutoff_fail = get_s_f(over_time_result,
cutoff_timestep)
# print("success={}, total={}, ratio={}".format(cutoff_success, cutoff_success + cutoff_fail, cutoff_success / (cutoff_success + cutoff_fail) * 100))
# for run, result in over_time_result.items():
# rate_list = []
# for region, req in result.items():
# rate_list.append(req.success / (req.success + req.fail))
# print(run, result, flush=True)
# print(run, np.mean(rate_list) * 100)
# avg_request.success += request_total.success
# avg_request.fail += request_total.fail
avg_request.success += cutoff_success
avg_request.fail += cutoff_fail
print("{}".format(cutoff_success /
(cutoff_fail + cutoff_success) * 100))
new_rate = get_success_ratio(instance_path)
# print("New success rate {}%".format(new_rate))
# print(request_total)
# print(region_result)
# print(over_time_result)
over_time_dict = {}
for run, region_dict in over_time_result.items():
over_time_dict[run] = Request.Request()
for region, request in region_dict.items():
over_time_dict[run].success += request.success
over_time_dict[run].fail += request.fail
# print(over_time_dict)
total = int(
(avg_request.success + avg_request.fail) / len(instances))
avg_request.success = int(avg_request.success / len(instances))
avg_request.fail = total - avg_request.success
print(avg_request)
def __init__(self, num_stages=4, stall_scheme='FREEZE'):
self.pipe_stage = []
self.num_stages = num_stages
self.stall_scheme = stall_scheme # SKID or FREEZE
self.stall = False
self.insert = Request()
self.insert_pend = False
for i in xrange(self.num_stages):
self.pipe_stage.append(Request())
def __init__(self,
sizeOfBuffer,
timeout,
numberOfThreads,
numberOfCores,
timeQuantum,
contextSwitchTime,
numberOfClients,
randomSeed,
arrivalTimeDistributionLambda,
thinkTimeDistribution,
serviceTimeDistribution,
paramThinkTime1,
paramServiceTime1,
paramThinkTime2=None,
paramServiceTime2=None):
random.seed(randomSeed)
self.eventList = EventList.EventList()
Request.Request.initRequestId()
self.simulationTime = 0
self.departureCount = 0
self.clients = []
for y in list(range(numberOfClients)):
self.clients.append(
Client.Client(y, thinkTimeDistribution, paramThinkTime1, 0,
paramThinkTime2)) #0 - thinking
self.requestList = RequestList.RequestList()
for index in list(range(numberOfClients)):
if serviceTimeDistribution == 1 or serviceTimeDistribution == 2: # Uniform or Normal distribution
request = Request.Request(index, arrivalTimeDistributionLambda,
serviceTimeDistribution, timeout,
paramServiceTime1, paramServiceTime2)
else:
request = Request.Request(index, arrivalTimeDistributionLambda,
serviceTimeDistribution, timeout,
paramServiceTime1)
self.requestList.addToRequestList(request)
#print (self.requestList.requestList[index].arrivalTime)
newEvent = Event.Event(self.simulationTime + request.arrivalTime,
0, request.requestId)
self.eventList.enqueueEvent(newEvent)
#schedule timeout of the request
newEvent1 = Event.Event(self.simulationTime + request.arrivalTime +
request.timeout, 4,
request.requestId) #4 - timeout
self.eventList.enqueueEvent(newEvent1)
self.system = System.System(sizeOfBuffer, numberOfCores,
numberOfThreads, timeQuantum,
contextSwitchTime)
self.metrics = Metrics.Metrics()
def main():
flag = True
while flag:
generalToken, flag = scanActivity()
jsonOrder = finalOrder()
print(jsonOrder)
Request.postBd(jsonOrder, generalToken)
# Barista.make(jsonOrder)
Visualizer.stopSession()
def __init__(self, depth=8):
self.occupancy = 0
self.depth = depth
self.storage = []
for i in xrange(self.depth):
self.storage.append(Request())
self.stats = []
self.push_pend_request = Request()
self.push_pend = False
self.pop_pend = False
def issueRequest(self):
if self.rate <= 0:
return
request = Request()
request.createdAt = self.sim.now
request.onCompleted = lambda: self.onCompleted(request)
self.server.request(request)
# Schedule the next one
self.scheduleRequest()
def finalOrder():
Request.postOrderStatus("WAITING")
timer = 2
while (Request.getOrderStatus() == "WAITING"):
time.sleep(timer)
timer += 2
if (timer >= 90):
timer = 1 # OK
return Request.getOrder()
def issueRequest(self): if self.rate <= 0: return request = Request() request.createdAt = self.sim.now request.onCompleted = lambda: self.onCompleted(request) self.server.request(request) # Schedule the next one self.scheduleRequest()
def main():
#parse params
serverHost = sys.argv[1]
serverPort = int(sys.argv[2])
path = sys.argv[3]
if (path == '/getImage' or path.startswith('/getImage?')):
#send image request
r = Request(serverHost, serverPort, 'GET', path)
print("[INFO] Sending request to %s:%s..." % (serverHost, serverPort))
start = time.time()
image = sendRequest(r, 1024 * 1024)
end = time.time()
print("[DEBUG] Got %d bytes" % (len(image)))
print("[DEBUG] Request received in %f seconds" % (end - start))
#write image file
try:
f = open('output.jpg', r'wb')
f.write(image)
print("[INFO] Image saved to %s" % ("output.jpg"))
except:
print("[ERROR] Failed to save image!")
finally:
f.close()
elif (path == '/captureAndDownload'):
#send image request
r = Request(serverHost, serverPort, 'GET', path)
print("[INFO] Sending request to %s:%s..." % (serverHost, serverPort))
start = time.time()
image = sendRequest(r, 1024 * 1024)
end = time.time()
print("[DEBUG] Got %d bytes" % (len(image)))
print("[DEBUG] Request received in %f seconds" % (end - start))
#write image file
try:
filename = 'img-%d' % (start)
f = open(filename + '.jpg', r'wb')
f.write(image)
print("[INFO] Image saved to %s" % (filename + '.jpg'))
except:
print("[ERROR] Failed to save image!")
finally:
f.close()
else:
start = time.time()
r = Request(serverHost, serverPort, 'POST', path)
outputText = sendRequest(r)
end = time.time()
print(outputText)
print("[DEBUG] Request received in %f seconds" % (end - start))
def ReplacementRequest(component, blood_type, units): #create replacer replacement = ReplacementRecord(0, component, blood_type, units, getCurrDate()+timedelta(days=7), 1) #replacementId = replacement.saveRR() ##########saves in db and returns replacement id #replacer = Replacer(0, "human", "addr", "125478963", getCurrDate() + timedelta(days=7) , replacementId, "instr") #---------------change------------------ #saveReplacerData() #create request req = Request(component, blood_type, units, 2) #serve request req.flow() #check stock level checkStockLevel(component, blood_type)
def decode(self, s):
"""Decode a string to a CRPRequest.
Args:
self: the decoder.
s: a string to decode.
"""
as_dictionary = super().decode(s)
# Decode the dictionary.
if as_dictionary["command"] == "upload":
data_dict = {
"status": json.loads(as_dictionary["data"]["status"]),
"images": as_dictionary["data"]["images"]
}
elif as_dictionary["command"] == "search":
data_dict = as_dictionary["data"]
else:
raise AttributeError()
return Request(as_dictionary['index'], as_dictionary['command'],
data_dict)
def __get(self, catch, save, log, url, sleep_time, times, timeout,
timeout_num):
data = []
sleeping_time = sleep_time
for i in range(1, times + 1):
response = Request.Request(url, timeout, timeout_num)
if response.result != 'Error':
if response.status == 200:
data_catch = catch.catch(response.result)
log.catch_log(url, len(data_catch))
data += data_catch
break
else:
if i < times:
sleeping_time = sleep_time * random.randint(1, i)
time.sleep(sleeping_time)
else:
log.catch_error_log(url,
'Status[%d]' % response.status)
else:
log.catch_error_log(url, response.e)
break
time.sleep(sleep_time)
num = save.save(data)
self.__lock.acquire()
self.__num = self.__num + num
self.__lock.release()
def client_handler(clientsock, addr):
global clients, client_id
curr_id = client_id
client_id += 1
clients[curr_id] = clientsock
buildings_string = pickle.dumps(buildings)
clientsock.send(buildings_string)
while True:
data = clientsock.recv(BUFFSIZE)
if 'Request:' not in data:
continue
else:
data = data.replace('Request:', '')
start, end = pickle.loads(data)
request = Request(curr_id, start, end)
requests.append(request)
print 'Added request, requests now:', requests
break
time.sleep(0.5)
while True:
if request in handled_requests:
taxi = request.taxi
if taxi.path:
taxi_info_tuple = (taxi.x, taxi.y, len(taxi.path), taxi.prev,
taxi.taxi_id)
#TODO - maybe pass taxi's path and then paint it in orange in client
taxi_string = pickle.dumps(taxi_info_tuple)
clientsock.send('Your taxi:' + taxi_string)
if (taxi.x, taxi.y) == end:
clientsock.send('Arrived to destination')
time.sleep(0.5)
def addGroup(group):
request = Req.Request(103)
content = {}
content['newgroup']=group
content['message']='Permintaan Tambah Group'
request.content = content
reqQueue.put(request.encode())
def process_input(self):
"""
In this function we will take assumptions listed below.
1. the file format selected is in .txt format
2. each line of text contains the flight request data in the following order
{ flightID, submissionTime, reqStart, reqDuration}
When program simulates a run there will be new values for actual takeoff times
{ flightID, submissionTime, reqStart, reqDuration, actualStart, actualEnd }
The scanner will read the file a line at a time. separated by a comma, at a time.
It'll instantiate a request object with relevant data read from a line of text.
It'll simply populate the list with Request objects
"""
print("========================Start of Process_Input() Method*")
request_data = ["name", 0, 0, 0] # initialing th object variables
req_data_counter = 0 # refers to an index in a list
with open(self.__file_name) as input_file:
whole_file = input_file.read().splitlines()
for i in range(len(whole_file)):
whole_file[i] = whole_file[i].split(',') # use comma as a delimiter
for j in range(len(whole_file[i])):
whole_file[i][j] = whole_file[i][j].strip()
if req_data_counter < 4: # we will break the data into units
request_data[req_data_counter] = whole_file[i][j]
req_data_counter = req_data_counter + 1
if req_data_counter > 3:
# create object, having read all values for a single req
new_request_object = Request.Request(request_data[0], request_data[1], request_data[2],
request_data[3])
self.input_list.append(new_request_object)
assert isinstance(new_request_object, object) # asserting if item added is object request
req_data_counter = 0 # resetting index counter to start reading new request data
print("========================file reading finished*")
self.display_contents(self.input_list)
print("========================End of Process_Input() Method *")
def removeGroup(group):
request = Req.Request(104)
content = {}
content['delgroup']=group
content['message']='Permintaan destroy Group'
request.content = content
reqQueue.put(request.encode())
def transport(self, target, data, **kw):
self.debug_msglog('initiating transport to target: %r' % target)
sid = None
if (kw.has_key('sid')):
sid = kw['sid']
request = Request(target)
request.add_header('Host', request.get_host())
contenttype = getattr(data, 'mimetype', self.content_type)
request.add_header('Content-Type', contenttype)
if not isinstance(data, str):
data = self._convert_to_string(data)
request.set_data(data)
return self.transaction_manager.create_transaction(request, sid)
def generate_request(self):
process_time = self.next_process_time()
timestamp = self.simulator.now
self.request_count += 1
self.upcoming = self.next_arrival()
new_request = Request.Request(self.request_count, process_time,
timestamp)
self.simulator.request_routing(new_request)
def run(self):
for b in self.__batch:
lines = b.split("\n")
for line in lines:
req = line.split(",")
self.__requests.append(
Request.Request(req[0], req[1], req[2], req[3], req[4]))
self.notify()
def download_file(self, url, folder, file_name):
download = Request.download(downloader=self,
url=url,
folder=folder,
file_name=file_name,
checksum=Config.CHECKSUM_FILE)
return download
def _download_data(server, token, channel, fmt, resolution, x_start, x_stop, y_start, y_stop, z_start, z_stop, location):
"""
Download the actual data from the server. Uses 1MB chunks when saving.
Returns the filename stored locally. Specify a save-location target in get_data.
"""
print("Downloading " + str(z_start) + "-" + str(z_stop))
# Build a string that holds the full URL to request.
req = Request(
server = server,
token = token,
channel = channel,
format = fmt,
resolution = resolution,
x_start = x_start,
x_stop = x_stop,
y_start = y_start,
y_stop = y_stop,
z_start = z_start,
z_stop = z_stop
)
request_url = req.to_url()
file_name = location + "/" + req.to_filename()
# Create a `requests` object.
req = requests.get(request_url, stream=True)
if req.status_code is not 200:
print(" !! Error encountered... Trying again in 5s...")
# Give the server five seconds to catch its breath
# TODO: ugh
time.sleep(5)
req2 = requests.get(request_url, stream=True)
if req2.status_code is not 200:
return (False, file_name)
else:
req = req2
# Now download (chunking to 1024 bytes from the stream)
with open(file_name, 'wb+') as f:
for chunk in req.iter_content(chunk_size=1024):
if chunk:
f.write(chunk)
f.flush()
return (True, file_name)
def build_request(self, url, data=None, headers={}, version='HTTP/1.1'):
request = Request(url, data, headers, version)
request.add_header('Host', request.get_host())
request.add_state_listener(self.completion_handler)
self.set_request(request)
self._request_count += 1
return request
def sendMessage(message,toGroup ='public',info=None):
request = Req.Request(201)
content = {}
content['sender']=USER_NAME
content['toGroup']=toGroup
content['message']=message
content['info']=info
request.content = content
reqQueue.put(request.encode())
def register(name="Anonymous",pic=""):
request = Req.Request(100)
content = {}
USER_NAME = name
content['name']=USER_NAME
content['profil']=pic
content['message']='Melakukan Register Awal'
request.content = content
reqQueue.put(request.encode())
def handle_route_on_status_update(bucket_name, object_name, status):
bucket_owner = BUCKET_TO_IP_DICT[bucket_name]
# Send post requests according to status
if status == '1':
req.handle_file_extract(bucket_name, object_name)
if status == '2':
# Convert files
bucket_content = req.handle_file_convert(bucket_name)
if status == '3':
# Pack files
req.handle_file_pack(bucket_name)
handle_status_update(bucket_name, '4')
return res.makeResponse(200, {
"status": "Success",
"bucketName": bucket_name
})
def getCommentIqiyi(cls, playlink, moviename):
if playlink != '':
source = Request.getSource(playlink)
if source is not None:
m = re.search(r'qitanid="\w+"', source)
vid = m.group(0).split('"')[1]
for page in range(1, 50):
jslink = 'http://api.t.iqiyi.com/qx_api/comment/get_video_comments?page='+str(page)+'&qitanid='+str(vid)
commentsource = Request.getSource(jslink)
pat = re.compile(r'content":"(.*?)"')
try:
commentlist = pat.findall(commentsource)
for comm in commentlist:
unescomm = pyunescape.pyunescape(comm)
if unescomm is not None and unescomm != "":
db.insert((moviename, 'iqiyi', unescomm))
except:
continue
def transport(self, target, data, **kw):
self.debug_msglog('initiating transport to target: %r' % target)
sid=None
if(kw.has_key('sid')):
sid=kw['sid']
request = Request(target)
request.add_header('Host', request.get_host())
contenttype = getattr(data, 'mimetype', self.content_type)
request.add_header('Content-Type', contenttype)
if not isinstance(data, str):
data = self._convert_to_string(data)
request.set_data(data)
return self.transaction_manager.create_transaction(request,sid)
def accept(self, warp):
distinct_lines = set()
requests = []
# assert(len(warp.instruction) == self.warp_width)
for i in range(len(warp.instruction)):
# split request into cache lines
lines = []
if warp.active_threads[i]:
lines = self.splitCacheLines(warp.instruction[i])
if len(lines) > 1 and lines[0][1] == "L":
# Notify the scoreboard that it needs to wait for
# extra requests to come back.
# Note we do this just for loads, not for stores,
# because stores are independent anyways.
warp.add_extra_completes(i, len(lines) - 1)
for line in lines:
distinct_lines.add(line)
# Create Requests
thread_requests = []
for line in lines:
r = Request(line[0], line[1], line[2], line[3])
# We need a separate copy for each so that the Requests
# can change the active threads without causing side-effects
w = copy.copy(warp)
w.active_threads = [False for x in warp.instruction]
w.active_threads[i] = True
r.addRequester(w)
thread_requests.append(r)
requests.append(thread_requests)
# assert(len(requests) == self.warp_width)
request = list(itertools.izip_longest(*requests))
self.queue += request
# Add number of lines to histogram
self.stats.increment("lines_per_warp_%d" % len(distinct_lines), 1)
self.stats.increment_average("lines_per_warp_avg", len(distinct_lines))
def resolveSohu(cls, movielink, moviename):
"""
sohu:从电影搜索页提取该搜索结果的播放页地址
"""
try:
f = Request.getSource(movielink)
source = BeautifulSoup(f)
links = source.find_all('a', title='点击观看')
playurl = links[0].get('href')
Comments.Comments.getCommentSohu(playurl, moviename)
except:
pass
def build_request(self, url, data=None, headers={}, version='HTTP/1.1'):
request = Request(url, data, headers, version)
request.add_header('Host', request.get_host())
request.add_state_listener(self.completion_handler)
self.set_request(request)
self._request_count += 1
return request
def getCommentSohu(cls, playlink, moviename):
if playlink != '':
source = Request.getSource(playlink)
if source is not None:
m = re.search(r'vid\s*=\s*[\"\']\w+[\"\']', source)
vid = m.group(0).split('"')[1]
n = re.search(r'playlistId\s*=\s*[\"\']\w+[\"\']', source)
playlistid = n.group(0).split('"')[1]
jslink = 'http://access.tv.sohu.com/reply/list/1000_' + \
str(playlistid) + '_' + str(vid) + '_0_2000.js'
commentsource = Request.getSource(jslink)
try:
comments = re.search(r'\[.*\]', commentsource)
comments = comments.group(0).replace("'",'"')
pat = re.compile(r'content":"(.*?)"')
commentlist = pat.findall(comments)
for comm in commentlist:
unescomm = pyunescape.pyunescape(comm)
if unescomm is not None and unescomm != "":
db.insert((moviename, 'sohu', unescomm))
except:
pass
def resolveFunshion(cls, movielink):
"""
funshion:从电影搜索页提取该搜索结果的播放页地址
"""
try:
f = Request.getSource(movielink)
source = BeautifulSoup(f)
links = source.find_all('ul', "search_list")
if(len(links) != 0):
playurl = 'http://www.funshion.com' + \
links[0].find('a').get('href')
print playurl
except:
print '从搜索结果提取播放地址失败'
def resolveTudou(cls, movielink):
"""
tudou:从电影搜索页提取该搜索结果的播放页地址
和优酷一样，都是从搜库提取数据，忽略之
"""
try:
f = Request.getSource(movielink)
source = BeautifulSoup(f)
links = source.find_all('div', "btnplay_s")
if(len(links) != 0):
playurl = links[0].find('a').get('href')
# if playurl[:7] != '/search':
print playurl
except:
print '从搜索结果提取播放地址失败'
def resolveYouku(cls, movielink):
"""
youku:从电影搜索页提取该搜索结果的播放页地址
去掉重定向到其他视频网站的链接，以及不存在的链接
"""
try:
f = Request.getSource(movielink)
source = BeautifulSoup(f)
links = source.find_all('div', "btnplay_s")
if(len(links) != 0):
playurl = links[0].find('a').get('href')
if playurl[:7] != '/search':
print playurl
except:
print '从搜索结果提取播放地址失败'
def getCommentLetv(cls, playlink, moviename):
if playlink != '':
vid = playlink.split('/')[-1].split('.')[0]
for i in range(1, 120):
jslink = 'http://api.my.letv.com/vcm/api/g?type=video&xid='+str(vid)+'&page='+str(i)
source = Request.getSource(jslink)
try:
pat = re.compile(r'content":"(.*?)"')
commentlist = pat.findall(source)
for comm in commentlist:
unescomm = pyunescape.pyunescape(comm)
if unescomm is not None and unescomm != "":
db.insert((moviename, 'letv', unescomm))
except:
continue
def getCommentFunshion(cls, playlink, moviename):
if playlink != '':
vid = playlink.split('/')[-2]
for page in range(1, 31):
jslink = 'http://q.funshion.com/ajax/get_comment/media/'+vid+'/all?pg='+str(page)
source = Request.getSource(jslink)
try:
pat = re.compile(r'content":"(.*?)"')
commentlist = pat.findall(source)
for comm in commentlist:
unescomm = pyunescape.pyunescape(comm)
if unescomm is not None and unescomm != "":
db.insert((moviename, 'funshion', unescomm))
except:
continue
def getCommentYouku(cls, playlink, moviename):
if playlink != '':
vid = playlink.split('_')[-1].split('.')[0]
for page in range(1, 100):
jslink = 'http://comments.youku.com/comments/~ajax/vpcommentContent.html?__ap={%22videoid%22:%22'+vid+'%22,'+'%22page%22:'+str(page)+'}'
commentsource = Request.getSource(jslink)
try:
pat = re.compile(r'content_(.*?)<br')
commentlist = pat.findall(commentsource)
for comm in commentlist:
comm = comm.split('">')
unescomm = pyunescape.pyunescape(comm[2])
if unescomm is not None and unescomm != "":
db.insert((moviename, 'youku', unescomm))
except:
continue
me.move(240)
me.move(270)
me.move(300)
me.move(330)
print("##################TEST des trames input#############################")
print("test MAP:")
obj = []
obj.append("GO:5:decor:2:50:2:{}")
obj.append("GO:5:decor:2:51:2:{}")
obj.append("GO:5:decor:2:52:2:{}")
obj.append("GO:5:decor:2:53:2:{}")
map_request = "1:1;3:map;6:player;0:;0:;" + obj[0] + "|" + obj[1] + "|" + obj[2] + "|" + obj[3]
map_request_tr = Trame.Trame([map_request])
Request.map(f, me, map_request_tr)
for i in me.core.map.gameObjects:
print("obj id: ", i.id, "is in map")
print("\ntest ALL_PLAYER:")
tpos = "CP:8:position:37:{i:1:0;i:1:0;i:1:0;i:1:0;i:1:0;i:1:0}"
tcol = "CP:7:colider:26:{i:1:0;i:1:0;i:1:0;i:2:20}"
tlife = "CP:4:life:9:{i:3:100}"
ttb = "CP:10:throw_bomb:2:{}"
tmv = "CP:4:move:2:{}"
tpsize = 1 + len(tpos) + 1 + len(tcol) + 1 + len(tlife) + 1 + len(ttb) + 1 + len(tmv) + 1
t_players = []
t_players.append("GO:6:player:1:0:" + str(tpsize) + ":{" + tpos + ";" + ttb + ";" + tlife + ";" + tcol + ";" + mv + "}")
t_players.append("GO:6:player:1:3:" + str(tpsize) + ":{" + tpos + ";" + ttb + ";" + tlife + ";" + tcol + ";" + mv + "}")
def HandleClient(clientsock):
import config
name = threading.currentThread().getName()
print name, ' Started.............'
global EOM
chunks = []
cmd=""
while 1:
buf = clientsock.recv(2048)
if len(buf)<6:
chunks[-1]+=buf
else:
chunks.append(str(buf))
if (EOM in chunks[-1]):
msg = "".join(chunks)[:-5]
if debug: print msg
if (msg == "TEST: HELLO"):
Response.sendData(clientsock,"Hello")
clientsock.close()
return
req = Request.parseRequest(msg)
if (not validReq(req)):
Response.sendData(clientsock, "Invalid Request")
print "invalid request"
clientsock.close()
return
if (req["requestType"]=="CLONE"):
job = Request.parseCloneJob(msg)
if job["scm"]=="git":
gclient=git.GIT(job["workdir"],job["repo"])
gclient.setKey(job["key"])
cmd=gclient.get_clone_cmd()
elif (req["requestType"] == "PULL"):
job = Request.parsePullJob(msg)
if job["scm"]=="git":
gclient=git.GIT(workdir=job["workdir"])
gclient.setKey(job["key"])
cmd=gclient.get_pull_cmd()
elif req["requestType"]=="LIST-TAGS":
job = Request.parseListTagsJob(msg)
if job["scm"]=="git":
gclient=git.GIT(workdir=job["workdir"])
gclient.setKey(job["key"])
cmd=gclient.get_list_tags_cmd()
result=[]
res=Common.run(cmd)
if "ERR:" in res:
Response.sendData(clientsock,res)
else:
for line in res.split("\n"):
try:
cmd="cd %s; git show %s"%(job["workdir"],line)
res=Common.run(cmd)
lines=res.split("diff --git ")[0]
info=lines.split("\n")
#print info
tag=info[0][4:]
tagger=info[1].split(": ")[1].split("<")[0].strip()
date=info[2].split(": ")[1].strip()
commit=info[6].split("commit ")[1]
result.append(",,".join([tag,tagger,date,commit]))
except:
pass
Response.sendData(clientsock,"\n".join(result))
return
elif req["requestType"] == "LIST-BRNACHS":
job = Request.parseListBranchsJob(msg)
if job["scm"] == "git":
gclient = git.GIT(workdir=job["workdir"])
cmd = gclient.get_list_branches()
result = []
res = Common.run(cmd)
if "ERR:" in res:
Response.sendData(clientsock, res)
else:
for line in res.split("\n"):
try:
if line!="":
result.append(line.replace("*","").strip())
except:
pass
#print result
Response.sendData(clientsock, "\n".join(result))
return
elif req["requestType"]=="LIST-COMMITS":
job = Request.parseGetCommitsJob(msg)
if job["scm"]=="git":
gclient=git.GIT(workdir=job["workdir"])
gclient.setKey(job["key"])
cmd=gclient.get_history_cmd(job["options"],limit=config.log_limit)
elif req["requestType"]=="SWITCH-TAG":
job = Request.parseSwitchTagJob(msg)
if job["scm"]=="git":
gclient=git.GIT(workdir=job["workdir"])
cmd=gclient.get_switch_to_tag_cmd(tag=job["tag"])
elif req["requestType"]=="SWITCH-COMMIT":
job = Request.parseSwitchCommitJob(msg)
if job["scm"]=="git":
gclient=git.GIT(workdir=job["workdir"])
cmd=gclient.switch_to_histroy_cmd(commit=job["commit"])
elif req["requestType"]=="DIFF-COMMIT":
job = Request.parseSwitchCommitJob(msg)
if job["scm"]=="git":
gclient=git.GIT(workdir=job["workdir"])
cmd=gclient.commit_diff_cmd(commit=job["commit"])
elif req["requestType"]=="LIST-CHANGES":
job = Request.parseGetChangeLog(msg)
if job["scm"] == "git":
gclient = git.GIT(workdir=job["workdir"])
cmd = gclient.get_changelog(since=job["options"]["since"], to=job["options"]["to"])
result = []
res = Common.run(cmd)
print res
if "ERR:" in res:
Response.sendData(clientsock, res)
else:
for line in res.split("\n"):
try:
if line != "":
result.append(line.replace("*", "").strip())
except:
pass
print result
Response.sendData(clientsock, "\n".join(result))
elif req["requestType"]=="DEPLOY":
print msg
job = Request.parseDeployJob(msg)
try:
config=yaml.safe_load(open(job["configFile"]))
autodeployer.deploy(config,job["workdir"])
res="Done"
except Exception as e:
res="ERR:"+traceback.format_exc()
if cmd!="":
print cmd
res=Common.run(cmd)
Response.sendData(clientsock,res)
if debug:
print "Ended,",res
else:
print "Ended"
clientsock.close()
break
import Request
from collections import OrderedDict
#Last.Fm
Api_Key = 'c513f3a2a2dad1d1a07021e181df1b1f'
ApiUrl = 'http://ws.audioscrobbler.com/2.0/?method={0}&api_key=' + Api_Key + '&format=json'
requestHandler = Request.requestHandler()
class Api:
Method_Artist_GetTopAlbums = 'artist.getTopAlbums'
Method_Album_GetInfo = 'album.getInfo'
Method_Track_Search = 'track.search'
def getUrlTopAlbumsByArtist(self,artist):
url = ApiUrl.replace('{0}',self.Method_Artist_GetTopAlbums) + '&artist=' + artist
return url
def getUrlTracksByArtistAndAlbum(self,artist,album):
url = ApiUrl.replace('{0}',self.Method_Album_GetInfo) + '&artist=' + artist + '&album=' + album
return url
def getUrlTracks(self,track):
url = ApiUrl.replace('{0}',self.Method_Track_Search) + '&track=' + track
return url
#Tracks
def getBysearch(self,search):
tracksDict = []
if search != "":
response = requestHandler.request(self.getUrlTracks(search))
jsonObject = requestHandler.encodeJson(response);
def put_data(token, channel, data, x_start, y_start, z_start, channel_type="image", server=DEFAULT_SERVER, x_stop=0, y_stop=0, z_stop=0, filename="tmp.hdf5"):
"""
Upload data onto the OCP server.
Arguments:
:server: ``string : ocpy.access.enums.DEFAULT_SERVER`` The server to access
:token: ``string`` The token to upload (must be read/write)
:channel: ``string`` The token to upload (must be read/write)
:data: ``numpy.ndarray`` The data to upload
:q_start: ``int`` Lower bound of Q dimension
:q_stop: ``int : 0`` Upper bound of Q dimension. If omitted, is
autopopulated to contain q_start + data-size.
:filename: A temporary HDF5 file to stream to the server.
Returns:
: bool : Success of the call (True/False).
"""
# Handle unset q_stops for dimension 'q'
if x_stop == 0: x_stop = x_start + data.shape[0]
if y_stop == 0: y_stop = y_start + data.shape[1]
if z_stop == 0: z_stop = z_start + data.shape[2]
# Throw exceptions if there has been a set dataset shape that
# is not matched by the shape of the data
if (x_stop - x_start) != data.shape[0]:
raise DataSizeError("Bad fit: x-range")
if (y_stop - y_start) != data.shape[1]:
raise DataSizeError("Bad fit: y-range")
if (z_stop - z_start) != data.shape[2]:
raise DataSizeError("Bad fit: z-range")
# TODO: Use h5py dataset casting
datatype = 'u' + data.dtype.name if data.dtype.name != 'int64' else 'uint32'
# Create an HDF5 file that holds the data in order to send it
fout = h5py.File(filename, driver="core", backing_store=True)
fout.create_dataset("CUTOUT", data.shape, data.dtype, compression="gzip", data=data)
fout.create_dataset("DATATYPE", data=datatype)
fout.create_dataset("CHANNELTYPE", data=channel_type)
fout.close()
# Create a request that holds the URL of the API endpoint
req = Request(
token = token,
channel = channel,
x_start = x_start,
x_stop = x_stop,
y_start = y_start,
y_stop = y_stop,
z_start = z_start,
z_stop = z_stop,
resolution = "1",
format = "hdf5"
)
url = req.to_url()
with open(filename, 'rb') as payload:
import pdb; pdb.set_trace()
req = requests.post(url, data=payload.read())
# If we return !200, clearly something went wrong...
if req.status_code == 200:
return True
else:
return req.status_code
