def fetch_page(page_url):
html = fetch_url(page_url)
html = html.decode()
# print( html)
match = re.findall(r'<img src="([^"]+.douban.com/view/group_topic/large/public/p\d+\.jpg)"', html)
if match is None:
print("no image in this page\n")
return False
file_table = kv.get("attitude")
page_images = kv.get("page_images")
image_page = kv.get("image_page")
for image_url in match:
file_name = image_url[image_url.rfind("/") + 1 :]
file_name = "images/" + file_name
image_page[file_name] = page_url
page_images.setdefault(page_url, []).append(file_name)
page_images[page_url] = list(set(page_images[page_url]))
kv.save("page_images", page_images)
kv.save("image_page", image_page)
if file_name in file_table:
if file_table[file_name] == "d":
print("hate", file_name)
continue
if os.path.isfile(file_name):
print("skip", file_name)
continue
print("save", file_name)
urllib.request.urlretrieve(image_url, file_name)
def fetch_page(page_url):
html = fetch_url(page_url)
html = html.decode()
# print( html)
match = re.findall(
r'<img src="([^"]+.douban.com/view/group_topic/large/public/p\d+\.jpg)"',
html)
if match is None:
print("no image in this page\n")
return False
file_table = kv.get('attitude')
page_images = kv.get('page_images')
image_page = kv.get('image_page')
for image_url in match:
file_name = image_url[image_url.rfind('/') + 1:]
file_name = 'images/' + file_name
image_page[file_name] = page_url
page_images.setdefault(page_url, []).append(file_name)
page_images[page_url] = list(set(page_images[page_url]))
kv.save('page_images', page_images)
kv.save('image_page', image_page)
if file_name in file_table:
if file_table[file_name] == 'd':
print('hate', file_name)
continue
if os.path.isfile(file_name):
print("skip", file_name)
continue
print("save", file_name)
urllib.request.urlretrieve(image_url, file_name)
def findNewest(key):
replica_req = []
current_max_value, current_max_VC, current_max_timestamp = kv.get(key)
for current_replica in META.REPLICAS:
if current_replica == IP_PORT:
continue
try:
A = requests.get("http://" + current_replica +
"/kv-store/verify/" + key, timeout=.5)
res = A.json()
replica_req.append(res)
except requests.exceptions.Timeout:
continue
temp_value = replica_req[-1]['value']
temp_VC = replica_req[-1]['causal_payload']
temp_timestamp = replica_req[-1]['timestamp']
compare_VC_results = fn.compareVC(temp_VC, current_max_VC)
# check if given value, if no value
if(temp_value is None):
continue
else:
no_result = False
# two VC's are the same, then do nothing
if fn.equalityVC(current_max_VC, temp_VC):
continue
if (compare_VC_results is None and current_timestamp > current_max_timestamp) or compare_VC_results:
(current_max_value, current_max_VC, current_max_timestamp) = (
temp_value, temp_VC, temp_timestamp)
return current_max_value, current_max_VC, current_max_timestamp
def get(key):
isKeyValid = fn.onlyKeyCheck(key)
client_CP = request.form.get('causal_payload')
if not client_CP or client_CP == "''":
client_CP = [0] * (META.REPLICAS_PER_PART)
else:
client_CP = fn.parseVC(client_CP)
if not isKeyValid[0]:
return fn.http_error(isKeyValid[1], isKeyValid[2])
if IS_REPLICA and fn.getPartitionId(key) == META.THIS_PARTITION:
current_max_value, current_max_VC, current_max_timestamp = findNewest(
key)
my_value, my_VC, my_timestamp = kv.get(key)
no_result = current_max_value is None
need_update = not fn.equalityVC(current_max_VC, my_VC)
status_code = 200
if need_update:
success, status_code = kv.put(
key, current_max_value, current_max_VC, current_max_timestamp)
if success:
message = {"result": "success", "value": current_max_value, "node_id": MY_ID,
"causal_payload": current_max_VC, "timestamp": current_max_timestamp}
return fn.http_success(message, status_code)
# The put fails then:
a, message, status_code = fn.keyCheck(key, value)[1]
return fn.http_error(message, status_code)
elif no_result:
return fn.http_error({"result": "error", "msg": "Key does not exist"}, 404)
message = {"result": "success", "value": current_max_value, "node_id": MY_ID,
"causal_payload": current_max_VC, "timestamp": current_max_timestamp}
return fn.http_error(message, status_code)
# Proxy
else:
correct_part = fn.getPartitionId(key)
for current_replica in META.GLOBAL_VIEW[correct_part]:
try:
A = requests.get("http://" + current_replica + "/kv-store/" +
key, data={'causal_payload': client_CP}, timeout=2)
status_code = A.status_code
response = A.json()
if status_code == 404:
return fn.http_error({"result": "error", "msg": "Key does not exist"}, 404)
message = {"result": response['result'], "value": response['value'], "partition_id": META.THIS_PARTITION,
"causal_payload": response['causal_payload'], "timestamp": response['timestamp']}
return fn.http_success(message, status_code)
except requests.exceptions.Timeout:
continue
return fn.http_error({"result": "Error", "msg": "Server unavailable"}, 500)
def dyn_register_device(productKey, productSecret, deviceName):
global on_dynreg_cb, device, deviceSecret
key = '_amp_customer_devicesecret'
deviceSecretdict = kv.get(key)
print("deviceSecretdict:", deviceSecretdict)
if isinstance(deviceSecretdict, str):
deviceSecret = deviceSecretdict
else:
deviceSecret = None
if deviceSecret is None:
key_info = {
'productKey': productKey,
'productSecret': productSecret,
'deviceName': deviceName
}
# 动态注册一个设备，获取设备的deviceSecret
device.register(key_info, on_dynreg_cb)
def rebalance():
for node_ipp in fn.getLocalView():
if node_ipp == META.IP_PORT:
continue
resp = gossip(node_ipp)
if resp == ERROR:
print("REBALANCE FAILED")
toSend = {}
for key in d:
correct_part = fn.getPartitionId(key)
if correct_part == META.THIS_PARTITION:
continue
curr_val, curr_vc, curr_ts = kv.get(key)
try:
toSend[correct_part].append({'key': key, 'value': curr_val, 'casual_payload': fn.listToString(
curr_vc), 'timestamp': curr_ts}) # raises Error the first time, disgusting!
except KeyError:
toSend[correct_part] = [{'key': key, 'value': curr_val, 'casual_payload': fn.listToString(
curr_vc), 'timestamp': curr_ts}]
kv.delete(key)
for partition_num in toSend:
for node_ipp in META.GLOBAL_VIEW[partition_num]:
try:
# convert content to string
content = json.dumps(toSend[partition_num])
resp = requests.put(
"http://" + node_ipp + "/key_dump", data={'content': content}, timeout=5)
break
except requests.exceptions.Timeout:
continue
d, vc, ts = kv.getDictionaries()
kv_data = {'d': json.dumps(d), 'vc': json.dumps(
vs), 'timestamp': json.dumps(ts)}
# send updated kvs to other replicas in partition
urls = ["http://" + node_ipp +
"/kv-store/duplicate" for node_ipp in fn.get_replicas(META.THIS_PARTITION)]
fn.put_broadcast(urls, kv_data)
def getGossip():
# Goes through all data, updates to newer data, and saves info about data that
# this node has that is more recent and sends that back
sent_d = json.loads(request.form.get('d'))
sent_vc = json.loads(request.form.get('vc'))
sent_timestamp = json.loads(request.form.get('timestamp'))
to_update = {}
for key in sent_d:
my_d_val, my_vc_val, my_ts_val = kv.get(key)
sent_d_val = sent_d[key]
sent_vc_val = sent_vc[key]
sent_ts_val = sent_timestamp[key]
# if sent has a value and we do not
if my_d_val is None:
result, status_code = kv.put(
key, sent_d_val, sent_vc_val, sent_ts_val)
continue
# if equal, do nothing
if fn.equalityVC(my_vc_val, sent_vc_val):
continue
compare_VC_results = fn.compareVC(my_vc_val, sent_vc_val)
if (compare_VC_results is None and my_ts_val > sent_ts_val) or compare_VC_results:
# mine is bigger, save it
to_update[key] = {"value": my_d_val,
"causal_payload": my_vc_val, "timestamp": my_ts_val}
else:
# theirs is bigger, update mine
result, status_code = kv.put(
key, sent_d_val, sent_vc_val, sent_ts_val)
return json.dumps(to_update)
import kv
kv_ret = kv.get('enableOneMinuteOnCloud')
if (kv_ret != 'disable'):
execfile('/lib/oneMinuteOnCloud.py')
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# apt-get install python3-tk
# pip-3.2 install Pillow
# apt-get install python3-pil
# sudo apt-get install python3-pil.imagetk
import os
import webbrowser
from PIL import Image
import kv
attitude = kv.get('attitude')
for f in attitude:
if len(attitude[f]) > 1:
print(attitude[f], '==>', attitude[f][0])
attitude[f] = attitude[f][0]
root = 'images'
files = os.listdir(root)
# print(files)
for f in files:
f = root + '/' + f
if f in attitude:
print('u', attitude[f], f)
continue
if not os.path.isfile(f):
print('warning', f, 'is not file')
continue
try:
img = Image.open(f)
# -*- coding: UTF-8 -*-
import os
import kv
import cgi
import json
import log
# enable debugging
import cgitb
cgitb.enable()
print("Content-Type: application/json;charset=utf-8\r")
print("\r")
root = os.getcwd()
attitude_map = kv.get('attitude')
post = cgi.FieldStorage()
action = post['action'].value
action = action[0]
f = post['file'].value
attitude_map[f] = action
file_path = root+'/'+f
if action == 'd' and os.path.isfile(file_path):
os.remove(file_path)
kv.save('attitude', attitude_map)
print(json.dumps({'code': 0, 'action':action, 'file': f}));
import os
import kv
import cgi
import json
import log
# enable debugging
import cgitb
cgitb.enable()
print("Content-Type: application/json;charset=utf-8\r")
print("\r")
root = os.getcwd()
attitude_map = kv.get("attitude")
post = cgi.FieldStorage()
action = post["action"].value
action = action[0]
f = post["file"].value
attitude_map[f] = action
file_path = root + "/" + f
if action == "d" and os.path.isfile(file_path):
os.remove(file_path)
kv.save("attitude", attitude_map)
print(json.dumps({"code": 0, "action": action, "file": f}))
# enable debugging
import cgitb
cgitb.enable()
print("Content-Type: text/html;charset=utf-8\r")
print("\r")
root = os.getcwd()
f = open(root+'/head.html')
print(f.read())
f.close()
image_root = root+'/images'
files = os.listdir(image_root)
image_page = kv.get('image_page')
attitude = kv.get('attitude')
get = cgi.FieldStorage()
QUERY_STRING = os.environ["QUERY_STRING"]
log.debug('QUERY_STRING', QUERY_STRING)
get = parse_qs(QUERY_STRING, True)
log.debug(str(parse_qs(QUERY_STRING, True)))
is_all = 'all' in get
log.debug('is_all', str(is_all))
file_time_list = [(f, image_root+'/'+f, os.path.getmtime(image_root+'/'+f)) for f in files]
if is_all:
file_time_list = sorted(file_time_list, key=operator.itemgetter(2), reverse=True)
else:
file_time_list = [x for x in file_time_list if time.time() - x[2] < 3600*24]
def stupidGet(key):
value, causal_payload, timestamp = kv.get(key)
message = {"value": value, "causal_payload": causal_payload,
"timestamp": timestamp}
return fn.http_success(message, 200)
def put(key):
if not request.json:
val = request.form.get('val')
client_CP = request.form.get('causal_payload')
client_timestamp = int(time.time())
if not client_CP or client_CP == "''":
client_CP = [0] * (META.REPLICAS_PER_PART)
else:
client_CP = fn.parseVC(client_CP)
# Check if key uses valid characters, value is small enough, etc.
isKeyValid = fn.keyCheck(key, val)
if not isKeyValid[0]:
return fn.http_error(isKeyValid[1], isKeyValid[2])
if IS_REPLICA and fn.getPartitionId(key) == META.THIS_PARTITION:
my_val, my_vc, my_ts = kv.get(key)
# Client has old CP
if fn.compare_payload(my_vc, client_CP) == LATEST:
message = {"result": "error", "causal_payload": fn.deparseVC(
client_CP), "msg": "Client payload not most recent, get again before trying again"}
return fn.http_error(message, 404)
else:
current_max_value, current_max_VC, current_max_timestamp = findNewest(
key)
if len(current_max_VC) < META.REPLICAS_PER_PART:
diff = len(META.REPLICAS_PER_PART) - len(current_max_VC)
current_max_VC = [0] * diff
compare_VC_results = fn.compare_payload(
current_max_VC, client_CP)
if fn.equalityVC(client_CP, current_max_VC):
# update value + increment
client_CP[fn.getNodeID(META.IP_PORT)] += 1
result, status_code = kv.put(
key, val, client_CP, client_timestamp)
message = {"result": "success", "value": val, "partition_id": META.THIS_PARTITION, "causal_payload": fn.deparseVC(
client_CP), "timestamp": str(client_timestamp)}
return fn.http_success(message, status_code)
elif (compare_VC_results == CONCURRENT and client_timestamp > int(current_max_timestamp)) or compare_VC_results == UPDATE:
# increment client_CP and update our value
client_CP[fn.getNodeID(META.IP_PORT)] += 1
result, status_code = kv.put(
key, val, client_CP, client_timestamp)
message = {"result": "success", "value": val, "partition_id": META.THIS_PARTITION, "causal_payload": fn.deparseVC(
client_CP), "timestamp": str(client_timestamp)}
return fn.http_success(message, status_code)
else:
# client is smaller: update our value and return failure
result, status_code = kv.put(
key, current_max_value, current_max_VC, current_max_timestamp)
message = {"result": "failure", "value": current_max_value, "partition_id": META.THIS_PARTITION,
"causal_payload": fn.deparseVC(client_CP), "timestamp": str(current_max_timestamp)}
return fn.http_error(message, status_code)
else:
correct_part = fn.getPartitionId(key)
send_data = {'causal_payload': client_CP, 'val': val}
for current_replica in META.GLOBAL_VIEW[correct_part]:
try:
A = requests.put("http://" + current_replica +
"/kv-store/" + key, data=send_data, timeout=2)
status_code = A.status_code
response = A.json()
message = {"result": response['result'], "value": response['value'], "partition_id": META.THIS_PARTITION,
"causal_payload": response['causal_payload'], "timestamp": response['timestamp']}
# this could be error or success
return fn.http_success(message, status_code)
except requests.exceptions.Timeout:
continue
return fn.http_error({"result": "Error", "msg": "Server unavailable"}, 500)
else:
return fn.http_error({"result": "Error", "msg": "No VALUE provided"}, 403)
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# apt-get install python3-tk
# pip-3.2 install Pillow
# apt-get install python3-pil
# sudo apt-get install python3-pil.imagetk
import os
import webbrowser
from PIL import Image
import kv
attitude = kv.get('attitude')
for f in attitude:
if len(attitude[f]) > 1:
print(attitude[f],'==>',attitude[f][0])
attitude[f] = attitude[f][0]
root = 'images'
files = os.listdir(root)
# print(files)
for f in files:
f = root+'/'+f
if f in attitude:
print('u',attitude[f],f)
continue
if not os.path.isfile(f):
print('warning', f, 'is not file')
continue
try:
img = Image.open(f)