def connect_wireless(self, l_rec_file, sensor):
conn_type = Priority(server_name=self.server_name,
server_id=self.server_id).conn_type()
if conn_type < 3:
if conn_type == 1:
print('On-board WiFi network available')
elif conn_type == 2:
print('Mobile network available')
data_gps = pd.read_csv(self.path + 'gps/gps_merged.csv')
gps_name = 'gps' + datetime.utcnow().strftime('%y%m%d') + '.csv'
data_gps.to_csv(self.path + 'logs/gps/' + gps_name, index=False)
try:
Transfer('/home/ec2-user/rtd/vessels/' + self.vessel_name +
'/').upload('logs/gps/' + gps_name, 'gps/' + gps_name)
except paramiko.ssh_exception.SSHException:
logging.debug('GPS data was not uploaded')
for file in l_rec_file:
print('/home/ec2-user/rtd/vessels/' + self.vessel_name + '/',
'logs/raw/{sensor}/'.format(sensor=sensor) + file)
Transfer('/home/ec2-user/rtd/vessels/' + self.vessel_name +
'/').upload(
'logs/raw/{sensor}/'.format(sensor=sensor) + file,
'sensor/{sensor}/'.format(sensor=sensor) + file)
def __init__(self):
super(User, self).__init__()
self.set("data", {})
self.set("priority", Priority())
self.set("share", Share())
self.set("role", None)
self.set("enabled", False)
def simulateArrivals(option):
"""Powers the simulation"""
simulateAdding = loadFromFile()
simulateAdding.sort(key=getKey)
ready = None
if option == 1:
ready = FCFS()
elif option == 2:
ready = Priority()
elif option == 3:
q = 2
try:
q = int(input("Enter Quantum (Defauult=2):"))
except:
print("Invalid entry. Using default Quantum of 2.")
pass
ready = RoundRobin(q)
elif option == 4:
runAllQueues()
return
elif option == 5:
exit(1)
else:
print("Invalid option.")
return
print("\r")
print("***************************************************")
print("Running %s Scheduler" % ready.type)
remainingQueueSize = 0
while len(simulateAdding) > 0 or remainingQueueSize > 0:
for k in simulateAdding[:]:
if k.simArrival <= ready.times.step:
try:
ready.add(
PCB(k.pid,
simArrival=k.simArrival,
simBurst=k.simBurst,
priority=k.priority))
except Exception as e:
print(e)
simulateAdding.remove(k)
print("Current Queue:")
ready.show()
remainingQueueSize = ready.runNew()
ready.finishQueue()
ready.printQueueSimStats(ready.type)
def cloud(self, ldata, sensor):
for filename, df in ldata:
if len(df) < 10: continue
conn_type = Priority(server_name=self.server_name,
server_id=self.server_id).conn_type()
if conn_type == 1 or conn_type == 2: # wifi
print('wifi or gsm aws')
Transfer('/home/ec2-user/rtd/vessels/{vess}/merged/{sensor}/'.
format(vess=setup_rtd.parameters['vessel_name'],
sensor=sensor)).upload(
'merged/{sensor}/'.format(sensor=sensor) +
filename, filename)
else:
logging.debug('There is no network available')
df.to_csv(self.path + 'queued/{sensor}'.format(sensor=sensor) +
filename,
index=False)
self.tow_num += 1
def adding_by_arrival_simulation():
simulateAdding = []
simulateAdding.append(PCB(9999, simArrival=0, simBurst=1))
for y in range(1, 5):
randPID = randint(1, 10000)
randArrive = (randint(1, 4))
randBurst = (randint(1, 9))
randPriority = (randint(1, 10))
simulateAdding.append(
PCB(randPID,
simArrival=randArrive,
simBurst=randBurst,
priority=randPriority))
#simulateAdding = []
simulateAdding.sort(key=getKey)
classGivenQueue = []
classGivenQueue.append(PCB(2710, simArrival=8, simBurst=6, priority=2))
classGivenQueue.append(PCB(2720, simArrival=8, simBurst=2, priority=1))
classGivenQueue.append(PCB(2730, simArrival=0, simBurst=8, priority=1))
classGivenQueue.append(PCB(2740, simArrival=2, simBurst=5, priority=3))
classGivenQueue.append(PCB(2750, simArrival=6, simBurst=10, priority=4))
classGivenQueue.sort(key=getKey)
simulateAdding = classGivenQueue
#ready = FCFS()
#521pm, FCFS is ok!!!
ready = Priority()
#620pm, priority is ok!!!
#ready = randPriority()
#ready = RoundRobin(1)
print("Pid, arrival, burst, priority")
for k in simulateAdding:
print(k.pid, k.simArrival, k.simBurst, k.priority)
j = 0
remainingQueueSize = 0
while len(simulateAdding) > 0 or remainingQueueSize > 0:
j += 1
for k in simulateAdding[:]:
if k.simArrival <= ready.times.step:
print("in sim...curr step:", ready.times.step)
if j == 1: position = 0
else: position = 1
print("In sim...adding: %s to position %s" % (k.pid, position))
try:
ready.add(
PCB(k.pid,
simArrival=k.simArrival,
simBurst=k.simBurst,
priority=k.priority)) #), position=position )
except Exception as e:
print(e)
#ready.show()
simulateAdding.remove(k)
remainingQueueSize = ready.runNew()
ready.finishQueue()
ready.printQueueSimStats("ready")
def test_peek_fail():
"""Test peek function on empty queue."""
from priority import Priority
new_p_q = Priority()
with pytest.raises(IndexError):
new_p_q.peek()
def test_init():
"""Test init function."""
from priority import Priority
new_p_q = Priority()
assert isinstance(new_p_q, Priority)
def test_peek():
"""Test peek function on populated queue."""
from priority import Priority
new_p_q = Priority([("monkey", 1), ("cat", 2), ("dog", 1)])
assert new_p_q.peek() == ("monkey")
def test_insert_01():
"""Test insert function to populated queue."""
from priority import Priority
new_p_q = Priority([("monkey", 1), ("dog", 1)])
new_p_q.insert("cat", 2)
assert new_p_q.priority_queue[2] == ["cat"]
def test_insert():
"""Test insert function to empty queue."""
from priority import Priority
new_p_q = Priority()
new_p_q.insert("monkey", 1)
assert new_p_q.priority_queue[1] == ["monkey"]
def create(request):
ctx = {}
if request.method == 'POST':
ctx['name'] = request.POST['name']
ctx['url'] = request.POST['url']
ctx['type'] = request.POST['type']
ctx['time'] = request.POST['time']
ctx['depth'] = request.POST['depth']
ctx['process'] = 0
ctx['host'] = request.POST['host']
ctx['priority'] = request.POST['priority']
ctx['mysql'] = request.POST['mysql']
ctx['redis'] = request.POST['redis']
if (ctx['type'] == "jd"): #京东
#r.set('jd_taskname',ctx['name'])
r.set('jd_totalpage', JDPara.totalpage)
r.set('jd_flag', JDPara.flag)
r.set('jd_isurl', JDPara.isurl)
r.set('jd_url', JDPara.url)
r.set('jd_wantpage', ctx['depth'])
if (ctx['type'] == "tb"): #淘宝
#r.set('tb_taskname',ctx['name'])
r.set('tb_isTaobao', TBPara.isTaobao)
r.set('tb_isdataSource', TBPara.isdataSource)
r.set('tb_isapiData', TBPara.isapiData)
r.set('tb_pageres', TBPara.pageres)
r.set('tb_resurl', TBPara.resurl)
r.set('tb_resurl2', TBPara.resurl2)
r.set('tb_url', TBPara.url)
r.set('tb_dataurl', TBPara.dataurl)
r.set('tb_wantpage', ctx['depth'])
if (ctx['type'] == "wy"): #网易新闻
#Para.c = ctx['blog_url']
r.set('wy_url', ctx['url'])
r.lpush('technewsspider:start_urls', ctx['url'])
r.set('wy_wantpage', ctx['depth'])
if (ctx['type'] == "csdn"): # 网易新闻
# Para.c = ctx['blog_url']
r.set('csdn_url', ctx['url'])
r.lpush('CSDNBlogCrawlSpider:start_urls', ctx['url'])
r.set('csdn_wantpage', ctx['depth'])
#r.set('wy_taskname',ctx['name'])
#print ctx['time']
# 计算优先级，同时对爬虫进行判断是否开启
pri = Priority()
level = pri.calculate(ctx['name'], ctx['type'], ctx['priority'],
ctx['time'], ctx['depth'])
#存入mysql数据库
task = Task.objects.get(name=ctx['name']) # 查询该条记录，任务名称不能重复
task.url = ctx['url'] # 修改
task.type = ctx['type']
task.time = ctx['time']
task.depth = ctx['depth']
task.process = ctx['process']
task.host = ctx['host']
task.priority = ctx['priority']
task.mysql = ctx['mysql']
task.redis = ctx['redis']
task.level = level
task.save() # 保存
#obj = models.Task(name=ctx['name'], url=ctx['url'],type=ctx['type'],time=ctx['time'],depth=ctx['depth'],
#process=ctx['process'],priority=ctx['priority'],mysql=ctx['mysql'],redis=ctx['redis'],level=level)
back = {'result': '任务创建成功'}
return JsonResponse(back)
def load_priorities(self):
priorities_rows = self.database.get_priorities()
for priority_row in priorities_rows:
priority = Priority(priority_row[1])
priority.id = priority_row[0]
self.priorities[priority.id] = priority
def create_priority(self, code):
priority = Priority(code)
self.database.insert_priority(priority)
self.priorities[priority.id] = priority
return priority
def set_priority(self, priority):
self.priority = Priority(priority)
def __init__(self, name, status, priority, description):
self.name = name
self.status = Status(status)
self.priority = Priority(priority)
self.description = description
self.id = id(self)
