class SpiderMain:
def __init__(self):
"""
初始化方法，主要是将其他组件实例化
"""
self.url_manager = UrlManager()
self.html_downloader = HtmlDownloader()
self.html_parser = HtmlParser()
self.data_storage = DataStorage()
def start(self):
"""
爬虫的主启动方法
:return:
"""
""" 页码 """
title = set()
for a in range(2, 10):
html = self.html_downloader.download(
'http://ggzy.foshan.gov.cn/jyxx/fss/zfcg_1108551/zbxx/index_'+str(a)+'.html?1')
_title = self.html_parser.titleParer(html)
for i in _title:
title.add(i)
for i in title:
print(i)
html = self.html_downloader.download(i)
_product = self.html_parser.contextParer(html)
self.data_storage.storage(_product)
def main(argv):
address = '127.0.0.1'
port = 9999
queryDB = ""
queryFields = None
usage = str.format('{0} -i <ip address> -p <port> -f <filter> -e <complex filter> -b <select db> -q <query fields> -l <list all dbs>',sys.argv[0])
try:
opts, args = getopt.getopt(argv,"hi:p:f:e:b:q:l",["help","filter=","address","port","complexFilter","listdb","queryFields","listAllDB"])
except getopt.GetoptError:
print(usage)
sys.exit(2)
for opt, arg in opts:
if opt == '-h':
print(usage)
sys.exit()
elif opt == "-i":
address = arg
elif opt == "-p":
port = int(arg)
elif opt in ("-f", "--filter"):
packet_log_config.setFilter(arg)
elif opt in ("-e"):
packet_log_config.setFilter(parseComplexFilter(arg))
elif opt in ("-b"):
queryDB = arg
elif opt in ("-q"):
queryFields = parseQueryFields(arg)
elif opt in ("-l"):
DataStorage.listDBs()
sys.exit()
if type(queryFields) is not dict:
proxy_instance = http_proxy.HttpProxy(address,port)
proxy_instance.start()
else:
HttpPacket.query(queryFields,queryDB)
def __init__(self):
"""
初始化方法，主要是将其他组件实例化
"""
self.url_manager = UrlManager()
self.html_downloader = HtmlDownloader()
self.html_parser = HtmlParser()
self.data_storage = DataStorage()
def test_basic_types(self):
ds = DataStorage.open('test_basic_types.h5', 'w')
test_int = 124
test_float = 0.1
test_1d_array = np.arange(1000000)
test_2d_array = np.random.rand(100, 100)
test_list = [1, 2, 3, 4]
test_dict = dict(int=123, float=111.1, list=test_list)
ds['int'] = test_int
ds['float'] = test_float
ds['1d_array'] = test_1d_array
ds['2d_array'] = test_2d_array
ds['list'] = test_list
ds['dict'] = test_dict
# Test return types
self.assertTrue(isinstance(ds['int'], int))
self.assertTrue(isinstance(ds['float'], float))
self.assertTrue(isinstance(ds['1d_array'], np.ndarray))
self.assertTrue(isinstance(ds['2d_array'], np.ndarray))
self.assertTrue(isinstance(ds['list'], collections.MutableSequence))
self.assertTrue(isinstance(ds['dict'], collections.MutableMapping))
# Test values of basic types
self.assertEqual(ds['int'], test_int)
self.assertEqual(ds['float'], test_float)
self.assertTrue(np.all(ds['1d_array'] == test_1d_array))
self.assertTrue(np.all(ds['2d_array'] == test_2d_array))
self.assertEqual(ds['list'], test_list)
self.assertEqual(ds['dict'], test_dict)
ds.flush()
# Test delete
del (ds['int'])
self.assertRaises(KeyError, self.getItem, ds, 'int')
del (ds['float'])
self.assertRaises(KeyError, self.getItem, ds, 'float')
del (ds['1d_array'])
self.assertRaises(KeyError, self.getItem, ds, '1d_array')
del (ds['2d_array'])
self.assertRaises(KeyError, self.getItem, ds, '2d_array')
del (ds['list'])
self.assertRaises(KeyError, self.getItem, ds, 'list')
del (ds['dict'])
self.assertRaises(KeyError, self.getItem, ds, 'dict')
ds.close()
# ds should be empty now
ds = DataStorage.open('test_basic_types.h5', 'r')
self.assertEqual(len(ds), 0)
ds.close()
def test_empty_arr(self):
arr = np.array([])
ds = DataStorage.open('test_empty_arr.h5', 'w')
ds['empty'] = arr
ds.close()
ds = DataStorage.open('test_empty_arr.h5', 'r')
self.assertTrue(np.all(arr == ds['empty']))
self.assertEqual(len(ds['empty']), 0)
def readCorpus(self, db_path, dev_path):
ds = DataStorage()
ds.load_db_from_file(db_path)
ds.load_valid_datasets(dev_path)
self.vec_prop_valid = ds.get_actual_input_features(
merged_col_name=True)
get_bert_tokens(self.vec_prop_valid, self._bert_tokenizer)
def getBumpCurrentSlope(simLabel, threshold=0):
fileName = 'bump_slope_data/bump_slope_detailed_noise_{0}.h5'.format(
simLabel)
msg = 'Loading bump vs. current slope from \'{0}\''.format(fileName)
log_info('getBumpCurrentlope (detailed)', msg)
ds = DataStorage.open(fileName, 'r')
slopes = ds['lineFitSlope'].flatten()
ds.close()
slopes[slopes < threshold] = np.nan
return slopes
def test_lists(self):
def appendListAndTest(ds, key, test_l, item):
ds[key].append(item)
test_l.append(item)
self.assertEqual(test_l, ds[key])
ds = DataStorage.open('test_lists.h5', 'w')
d1 = {"hola": [10, 20, 30], "str": "This is a test string"}
test_list = [1, 2, d1]
ds['list'] = test_list
ds.close()
ds = DataStorage.open('test_lists.h5', 'r+')
appendListAndTest(ds, 'list', test_list, 10)
appendListAndTest(ds, 'list', test_list, 23.5)
appendListAndTest(ds, 'list', test_list, [1, 2, 3])
appendListAndTest(ds, 'list', test_list, dict(a=10, b=[1, 2, 3]))
ds.close()
class SpiderMain:
def __init__(self):
self.url_manager = UrlManager()
self.html_downloader = HtmlDownloader()
self.html_parser = HtmlParser()
self.data_storage = DataStorage()
def start(self):
"""
爬虫的主启动方法
:return:
"""
self.url_manager.add_new_url(
"http://127.0.0.1:8848/xiaomi-master/index.html")
# 从url管理器获取url
url = self.url_manager.get_new_url()
# 将获取到的url使用下载器进行下载
html = self.html_downloader.download(url)
# 将html进行解析
res = self.html_parser.parser(html)
# 数据存储
self.data_storage.storage(res)
def test_iterator(self):
ds = DataStorage.open('test_iterator.h5', 'w')
test_list = list(np.arange(100))
ds['list'] = test_list
it1 = 0
for val1 in ds['list']:
self.assertEqual(val1, test_list[it1])
it1 += 1
it2 = 0
for val2 in ds['list']:
self.assertEqual(val2, test_list[it2])
it2 += 1
ds.close()
def test_chained_setter(self):
def testChain(ds, keyList, testValue):
val = ds
for key in keyList[0:-1]:
val = val[key]
self.assertTrue(isinstance(val, collections.MutableMapping))
self.assertEqual(val[keyList[-1]], testValue)
keyList = ['a', 'b', 'c', 'd']
testValue = [1, 2, 3, 4]
ds = DataStorage.open('test_chained_setter.h5', 'w')
# Initial test
ds.setItemChained(keyList, testValue)
testChain(ds, keyList, testValue)
# Over-write test
newTestValue = 'Over-write test string'
ds.setItemChained(keyList, newTestValue)
testChain(ds, keyList, newTestValue)
# Do not overwrite if overwriteLast is False
noOverWriteTestValue = 'This should not be written'
ds.setItemChained(keyList, noOverWriteTestValue, overwriteLast=False)
testChain(ds, keyList, newTestValue)
# ['a', 'b', 'another', 'xxx']
otherKeyList = ['a', 'b', 'another', 'xxx']
otherTestValue = [1, 2, 3, 'ahoy']
ds.setItemChained(otherKeyList, otherTestValue)
testChain(ds, otherKeyList, otherTestValue)
testChain(ds, keyList, newTestValue) # Should not be overwritten
# Single item in keyList
singleList = ['single']
ds.setItemChained(singleList, otherTestValue)
testChain(ds, singleList, otherTestValue)
# Test using getItemChained()
self.assertEqual(ds.getItemChained(keyList), newTestValue)
self.assertEqual(ds.getItemChained(otherKeyList), otherTestValue)
self.assertEqual(ds.getItemChained(singleList), otherTestValue)
def test_chained_getter(self):
test_dict = dict(
int=123,
float=111.1,
list=[1, 2, 3,
dict(a='blabla', b=10, c=np.random.rand(10))])
ds = DataStorage.open('test_chained_getter.h5', 'w')
ds['nested'] = test_dict
ds_nested = ds['nested']
self.assertEqual(ds_nested.getItemChained(('int', )), test_dict['int'])
self.assertEqual(ds_nested.getItemChained(('list', 0)),
test_dict['list'][0])
self.assertEqual(ds_nested.getItemChained(('list', 3, 'a')),
test_dict['list'][3]['a'])
# The same test but with a list as an index
self.assertEqual(ds_nested.getItemChained(['list', 3, 'a']),
test_dict['list'][3]['a'])
ds.close()
def __init__(self, bert_tokenizer=None, bert_config=None):
self.ds = DataStorage()
self._bert_tokenizer = bert_tokenizer
self._bert_config = bert_config
class SQLTrainer:
def __init__(self, bert_tokenizer=None, bert_config=None):
self.ds = DataStorage()
self._bert_tokenizer = bert_tokenizer
self._bert_config = bert_config
def readCorpus(self, db_path, vec_train_path, dev_path):
self.ds.load_db_from_file(db_path)
self.ds.load_train_datasets(vec_train_path, dev_path)
# 1. Generate property-specific train / test data.
self.vec_prop_train, self.vec_prop_valid = self.ds.get_prop_cla_features(
merged_col_name=True)
self.vec_real_valid = self.ds.get_actual_input_features(
merged_col_name=True)
get_bert_tokens(self.vec_prop_train + self.vec_prop_valid,
self._bert_tokenizer)
train_bert_removed = 0
valid_bert_removed = 0
final_train = []
final_valid = []
for v_f in self.vec_prop_train:
bert_len = 0
bert_len += 2 + len(v_f[Q_BERT_TOK]) + 1 + len(v_f[PF_PATH])
for vec_tbl_bert in v_f[C_BERT_TOK]:
for vec_col_bert in vec_tbl_bert:
bert_len += 1 + len(vec_col_bert)
bert_len += 2 # *
if bert_len >= 512:
train_bert_removed += 1
else:
final_train.append(v_f)
for v_f in self.vec_prop_valid:
bert_len = 0
bert_len += 2 + len(v_f[Q_BERT_TOK]) + 1 + len(v_f[PF_PATH])
for vec_tbl_bert in v_f[C_BERT_TOK]:
for vec_col_bert in vec_tbl_bert:
bert_len += 1 + len(vec_col_bert)
bert_len += 2 # *
if bert_len >= 512:
valid_bert_removed += 1
else:
final_valid.append(v_f)
self.vec_prop_train = final_train
self.vec_prop_valid = final_valid
print("TOO LARGE FOR BERT (TRAIN): ", train_bert_removed)
print("TOO LARGE FOR BERT (VALID): ", valid_bert_removed)
get_bert_tokens(self.vec_real_valid, self._bert_tokenizer)
self.sg = SQLGen(bert_config=self._bert_config)
# Extract Path IDX.
for v_f in self.vec_prop_train + self.vec_prop_valid:
v_f[PF_PATHIDX] = [IDX_PATH[str_path] for str_path in v_f[PF_PATH]]
# Extract Actually used tables.
# Remove those required to JOIN, reducing the garbage inputs.
train_reduce_num = 0
valid_reduce_num = 0
vec_filter_codes = [ [ OF_VU_COL1, OF_VU_OPERATOR, OF_VU_COL2 ], [ SF_VU_COL1, SF_VU_OPERATOR, SF_VU_COL2 ], \
[ WF_CU_VU_COL1, WF_CU_VU_OPERATOR, WF_CU_VU_COL2 ], [ HV_CU_VU_COL1, HV_CU_VU_OPERATOR, WF_CU_VU_COL2 ] ]
for v_f in self.vec_prop_train + self.vec_prop_valid:
if 0 not in v_f[SF_VU_COL1]: # Try to reduce. ( NO STAR )
set_used_col_idx = set()
for c1_code, op_code, c2_code in vec_filter_codes:
for col1, op, col2 in zip(v_f[c1_code], v_f[op_code],
v_f[c2_code]):
set_used_col_idx.add(col1)
if op > 0:
set_used_col_idx.add(col2)
db = v_f[META_DB]
set_used_tbl_idx = set()
for cidx in set_used_col_idx:
if db.vec_cols[cidx].table_belong != None:
set_used_tbl_idx.add(
db.vec_cols[cidx].table_belong.tbl_idx)
if set(v_f[TV_TABLES_USED_IDX]) != set_used_tbl_idx:
recover = self._generate_actual_tbls(
db, list(set_used_tbl_idx))
if set(recover) == set(v_f[TV_TABLES_USED_IDX]):
if v_f in self.vec_prop_train:
train_reduce_num += 1
else:
valid_reduce_num += 1
v_f[TV_TABLES_USED_IDX] = list(set_used_tbl_idx)
v_f[TV_TABLES_NUM] = len(v_f[TV_TABLES_USED_IDX])
print("TRAIN REDUCE NUM: [%d]" % train_reduce_num)
print("VALID REDUCE NUM: [%d]" % valid_reduce_num)
print("PROPERTY TRAIN INSTANCE NUM: [%d]" % len(self.vec_prop_train))
print("PROPERTY VALID INSTANCE NUM: [%d]" % len(self.vec_prop_valid))
# Test on the features.
def _do_test(self, sess, target_data, batch_size):
batch_num = int(len(target_data) / batch_size)
if len(target_data) % batch_size != 0:
batch_num += 1
ti = TestInfo()
add_prop_test_info(ti, self.sg)
add_tbl_test_info(ti, self.sg)
for batch_idx in range(batch_num):
vec_data = target_data[batch_idx * batch_size:min(
(batch_idx + 1) * batch_size, len(target_data))]
# 1. Classify Tbl.
fdv_tbl = prepare_tbl_dict(vec_data,
bert_tokenizer=self._bert_tokenizer)
fdv_tbl[self.sg.is_train] = False
fdv_tbl[self.sg.drop_rate] = 0.0
tbl_results = ti.fetch_tbl_tensor_info(sess, fdv_tbl)
# 2. Get the extracted tbl info.
vec_tbl_idx_score = tbl_results[TV_TABLES_USED_IDX]
vec_tbl_num = tbl_results[TV_TABLES_NUM]
vec_mod_tbl_num = []
vec_tbl_extracted = []
for tbl_idx_score, tbl_num in zip(vec_tbl_idx_score, vec_tbl_num):
vec_tbl_idx = np.array(tbl_idx_score).argsort()[-tbl_num:]
vec_tbl_idx = sorted(vec_tbl_idx.tolist())
vec_tbl_extracted.append(vec_tbl_idx)
# 3. Classify Props.
fdv_prop = prepare_prop_dict(vec_data,
vec_tbl_extracted,
bert_tokenizer=self._bert_tokenizer)
fdv_prop[self.sg.is_train] = False
fdv_prop[self.sg.drop_rate] = 0.0
prop_results = ti.fetch_prop_tensor_info(sess, fdv_prop)
# 4. Update Pointer results to the actual columns.
vec_score_answers = [ [ GF_NUMCOL, [ GF_COLLIST ] ], \
[ OF_NUMVU, [ OF_VU_COL1, OF_VU_COL2, OF_VU_AGG1, OF_VU_AGG2, OF_VU_DIST1, OF_VU_DIST2, OF_VU_OPERATOR ] ], \
[ SF_NUM_VU, [ SF_VU_AGGALL, SF_VU_COL1, SF_VU_COL2, SF_VU_AGG1, SF_VU_AGG2, SF_VU_DIST1, SF_VU_DIST2, SF_VU_OPERATOR ] ], \
[ WF_NUM_CONDUNIT, [ WF_CU_AGGREGATOR, WF_CU_IS_NOT, WF_CU_COND_OP, \
WF_CU_VAL1_TYPE, WF_CU_VAL1_SP, WF_CU_VAL1_EP, WF_CU_VAL1_LIKELY, WF_CU_VAL1_BOOLVAL, \
WF_CU_VAL2_TYPE, WF_CU_VAL2_SP, WF_CU_VAL2_EP, WF_CU_VAL2_LIKELY, WF_CU_VAL2_BOOLVAL, \
WF_CU_VU_OPERATOR, WF_CU_VU_AGG1, WF_CU_VU_COL1, WF_CU_VU_DIST1, WF_CU_VU_AGG2, WF_CU_VU_COL2, WF_CU_VU_DIST2 ] ], \
[ HV_NUM_CONDUNIT, [ HV_CU_AGGREGATOR, HV_CU_IS_NOT, HV_CU_COND_OP, \
HV_CU_VAL1_TYPE, HV_CU_VAL1_SP, HV_CU_VAL1_EP, HV_CU_VAL1_LIKELY, HV_CU_VAL1_BOOLVAL, \
HV_CU_VAL2_TYPE, HV_CU_VAL2_SP, HV_CU_VAL2_EP, HV_CU_VAL2_LIKELY, HV_CU_VAL2_BOOLVAL, \
HV_CU_VU_OPERATOR, HV_CU_VU_AGG1, HV_CU_VU_COL1, HV_CU_VU_DIST1, HV_CU_VU_AGG2, HV_CU_VU_COL2, HV_CU_VU_DIST2 ] ] ]
for num_col_name, vec_col_name in vec_score_answers:
for col_name in vec_col_name:
prop_results[col_name] = [
np.argmax(v, -1).tolist()[:l] for v, l in zip(
prop_results[col_name], prop_results[num_col_name])
]
# 5. Update the extracted column indexes
vec_col_pointers = [
GF_COLLIST, OF_VU_COL1, OF_VU_COL2, SF_VU_COL1, SF_VU_COL2,
WF_CU_VU_COL1, WF_CU_VU_COL2, HV_CU_VU_COL1, HV_CU_VU_COL2
]
for pidx, prop_data in enumerate(vec_data):
db = prop_data[META_DB]
# 1. Get the valid column span. ( Inclusive )
vec_col_spans = [[0, 0]] # Special colum: *
for t in vec_tbl_extracted[pidx]:
if t < len(db.vec_tbls):
tbl = db.vec_tbls[t]
vec_col_spans.append([
tbl.vec_cols[0].col_idx, tbl.vec_cols[-1].col_idx
])
else:
print("ERROR: %d VS. %d" % (t, len(db.vec_tbls)))
vec_col_spans = sorted(vec_col_spans, key=lambda x: x[0])
# 2. Generate New Col Idx - Old Col Idx Map.
new_idx = 0
map_col_idx = dict()
for sidx, eidx in vec_col_spans:
for col_idx in range(sidx, eidx + 1):
map_col_idx[new_idx] = col_idx
new_idx += 1
# 3. Update the mappings.
for col_name in vec_col_pointers:
prop_results[col_name][pidx] = [
map_col_idx[col_idx]
for col_idx in prop_results[col_name][pidx]
if col_idx in map_col_idx
]
# 6. Get merged results & evaluate.
total_results = prop_results
for k, v in tbl_results.items():
total_results[k] = v
ti.integrate_eval_result(total_results, vec_data)
return ti
def train(self, BERT_DIR, lf, save_path, batch_size):
batch_num = int(len(self.vec_prop_train) / batch_size)
if len(self.vec_prop_train) % batch_size != 0:
batch_num += 1
self.sg.constructGraph(batch_num,
lf,
init_bert=os.path.join(BERT_DIR,
"bert_model.ckpt"))
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
with tf.Session(graph=self.sg.graph, config=config) as sess:
ckpt = tf.train.get_checkpoint_state(save_path)
v2_path = ckpt.model_checkpoint_path + ".index" if ckpt else ""
self.sg.init.run()
tvars = tf.trainable_variables()
initialized_variable_names = {}
(assignment_map, initialized_variable_names
) = modeling.get_assignment_map_from_checkpoint(
tvars, os.path.join(BERT_DIR, "bert_model.ckpt"))
tf.logging.info("**** Trainable Variables ****")
for var in tvars:
init_string = ""
if var.name in initialized_variable_names:
init_string = ", *INIT_FROM_CKPT*"
print(" name = %s, shape = %s%s" %
(var.name, var.shape, init_string))
print('Num params: %d' % sum(v.get_shape().num_elements()
for v in tf.trainable_variables()))
step = 1
valid_accept_cnt = 0
global_step = 0
prev_valid_acc = 0.0
while True:
self.vec_prop_train = corpus_util.shuffleData(
self.vec_prop_train) # JUST FOR DEBUGGING
print("STEP: %d" % step)
average_train_loss = 0.0
average_valid_loss = 0.0
tm = TimeMeasurer()
tm.start("train")
for batch_idx in range(batch_num):
global_step += 1
vec_prop_data = self.vec_prop_train[
batch_idx *
batch_size:min((batch_idx + 1) *
batch_size, len(self.vec_prop_train))]
# 1. Train the Table Selector Network.
feed_dict_train = prepare_tbl_dict(
vec_prop_data, bert_tokenizer=self._bert_tokenizer)
feed_dict_train[self.sg.global_step] = global_step
feed_dict_train[self.sg.is_train] = True
feed_dict_train[self.sg.drop_rate] = 0.1
_, loss_val = sess.run(
[self.sg.train_tbl_op, self.sg.tbl_cross_entropy],
feed_dict=feed_dict_train)
average_train_loss += loss_val * len(vec_prop_data)
if math.isnan(loss_val) or math.isinf(loss_val):
print("TBL NAN/INF ERROR")
sys.exit(0)
# 2. Train for Other property selection network.
# Feed the used tables as its inputs.
vec_valid_tbl_idx = [
v_f[TV_TABLES_USED_IDX] for v_f in vec_prop_data
]
feed_dict_prop_train = prepare_prop_dict(
vec_prop_data,
vec_valid_tbl_idx,
bert_tokenizer=self._bert_tokenizer)
feed_dict_prop_train[self.sg.global_step] = global_step
feed_dict_prop_train[self.sg.is_train] = True
feed_dict_prop_train[self.sg.drop_rate] = 0.1
_, loss_val = sess.run(
[self.sg.train_op, self.sg.cross_entropy],
feed_dict=feed_dict_prop_train)
average_train_loss += loss_val * len(vec_prop_data)
if math.isnan(loss_val) or math.isinf(loss_val):
print("PROP NAN/INF ERROR")
sys.exit(0)
if batch_idx % 100 == 0:
print("%d, Train Loss: %f" % (batch_idx, loss_val))
print("%d, Train Loss: %f" % (batch_idx, loss_val))
average_train_loss /= len(self.vec_prop_train)
tm.end("train")
print("Average Training Loss: " + str(average_train_loss))
print("Elapsed Per Step: " + str(tm.getElapsed("train")))
ti = self._do_test(sess, self.vec_prop_valid, batch_size)
ti.print_eval_results()
em_f1 = ti.get_overall_result()
valid_acc = em_f1
if valid_acc < prev_valid_acc:
if valid_accept_cnt >= 20:
print("Training Finished.")
return
else:
valid_accept_cnt += 1
print("TOLERATING CNT: %d" % valid_accept_cnt)
else:
self._save_model(sess, save_path)
print("MODEL SAVED.")
prev_valid_acc = valid_acc
valid_accept_cnt = 0
step += 1
def test(self, load_path, batch_size):
self.sg.constructGraph()
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.allow_soft_placement = True
with tf.Session(graph=self.sg.graph, config=config) as sess:
self.sg.init.run()
self._load_model(sess, load_path)
tm = TimeMeasurer()
tm.start("test")
ti = self._do_test(sess, self.vec_prop_valid, batch_size)
tm.end("test")
elapsed = tm.getElapsed("test")
print("elased_inmsec = " + str(elapsed) + "( per sen: " +
str(float(elapsed) / len(self.vec_prop_valid)) + " )")
ti.print_eval_results()
def _save_model(self, sess, save_path):
self.sg.saver.save(sess, "%s/best_chk" % save_path)
def _load_model(self, sess, save_path):
ckpt = tf.train.get_checkpoint_state(save_path)
v2_path = ckpt.model_checkpoint_path + ".index" if ckpt else ""
self.sg.saver.restore(sess, ckpt.model_checkpoint_path)
def _generate_actual_tbls(self, db, vec_tbl_idx):
if len(vec_tbl_idx) == 1:
return vec_tbl_idx
# 1. Find the "Join Path".
joinable_tbls = dict()
for f1, f2 in db.foreign_keys:
t1 = f1.table_belong.tbl_idx
t2 = f2.table_belong.tbl_idx
if t1 not in joinable_tbls:
joinable_tbls[t1] = set()
if t2 not in joinable_tbls:
joinable_tbls[t2] = set()
joinable_tbls[t1].add(t2)
joinable_tbls[t2].add(t1)
cur_path_cand = [set([v]) for v in vec_tbl_idx]
final_path = None
while len(cur_path_cand) > 0:
new_path_cand = []
for path_cand in cur_path_cand:
for t in path_cand:
if t not in joinable_tbls:
continue
for new_exp in joinable_tbls[t]:
if new_exp not in path_cand:
new_path = path_cand.copy()
new_path.add(new_exp)
if set(vec_tbl_idx) <= new_path:
final_path = new_path
break
if new_path not in new_path_cand:
new_path_cand.append(set(new_path))
if final_path != None:
break
if final_path != None:
break
if final_path != None:
break
cur_path_cand = new_path_cand
if final_path == None:
return vec_tbl_idx
return final_path
def main():
# data storage will parse all failures from LANL data file which contains failure trace logs
data_storage = DataStorage(
"../data/LA-UR-05-7318-failure-data-1996-2005.csv",
DatasetType.LANL_DATASET)
number_of_samplings = 10000
number_of_app_executions = 40
ec_node_candidate_list = [(19, 1), (19, 11), (19, 4), (19, 8), (20, 41)]
ed_node_candidate_list = [(1, 0), (5, 158), (5, 165), (5, 243), (5, 48),
(7, 1), (7, 154), (7, 242), (7, 32)]
er_node_candidate_list = [(3, 0), (16, 80), (4, 55), (4, 1), (4, 3)]
cd_node_candidate_list = [(22, 0)]
# ec_node_candidate_list = [(19, 8)]
# ed_node_candidate_list = [(5, 243)]
# er_node_candidate_list = [(4, 1)]
# cd_node_candidate_list = [(22, 0)]
application_names = [
'ANTIVIRUS', 'GPS_NAVIGATOR', 'CHESS', 'FACEBOOK', 'FACERECOGNIZER'
]
mixed_mode = True
sensitivity_analysis = False
# NodeCategory.EC_DATA: [(19, 1)],
# NodeCategory.ED_DATA: [(1, 0)],
# NodeCategory.ER_DATA: [(3, 0)],
# NodeCategory.CD_DATA: [(22, 0)]
#}
for i in range(5):
# declare mobile device with offloading sites included in the simulation and data provider
ec_node_candidate = ec_node_candidate_list[i]
ed_node_candidate = ed_node_candidate_list[i]
er_node_candidate = er_node_candidate_list[i]
cd_node_candidate = cd_node_candidate_list[0]
application = application_names[i]
print('EC node candidate: ' + str(ec_node_candidate))
print('ED node candidate: ' + str(ed_node_candidate))
print('ER node candidate: ' + str(er_node_candidate))
print('CD node candidate: ' + str(cd_node_candidate))
edge_data_site = OffloadingSite(
5000, 8, 300, OffloadingSiteCode.EDGE_DATABASE_SERVER,
data_storage.get_ed_data_stats(), 'A', ed_node_candidate)
edge_comp_site = OffloadingSite(
8000, 8, 150,
OffloadingSiteCode.EDGE_COMPUTATIONAL_INTENSIVE_SERVER,
data_storage.get_ec_data_stats(), 'A', ec_node_candidate)
edge_reg_site = OffloadingSite(5000, 8, 150,
OffloadingSiteCode.EDGE_REGULAR_SERVER,
data_storage.get_er_data_stats(), 'A',
er_node_candidate)
cloud_dc_site = OffloadingSite(12000, 128, 1000,
OffloadingSiteCode.CLOUD_DATA_CENTER,
data_storage.get_cd_data_stats(), 1,
cd_node_candidate)
mobile_device = MobileDevice(
(edge_data_site, edge_comp_site, edge_reg_site), cloud_dc_site)
mobile_device.deploy_network_model()
if not mixed_mode:
if application == 'ANTIVIRUS':
mobile_device.deploy_antivirus_application()
elif application == 'GPS_NAVIGATOR':
mobile_device.deploy_gps_navigator_application()
elif application == 'CHESS':
mobile_device.deploy_chess_application()
elif application == 'FACEBOOK':
mobile_device.deploy_facebook_application()
elif application == 'FACERECOGNIZER':
mobile_device.deploy_facerecognizer_application()
mobile_device.deploy_enhanced_efpo_ode()
mobile_device.run(number_of_samplings, number_of_app_executions,
mixed_mode, sensitivity_analysis)
mobile_device.deploy_efpo_ode()
mobile_device.run(number_of_samplings, number_of_app_executions,
mixed_mode, sensitivity_analysis)
mobile_device.deploy_energy_efficient_ode()
mobile_device.run(number_of_samplings, number_of_app_executions,
mixed_mode, sensitivity_analysis)
def __init__(self,):
super(DataFetcher, self).__init__()
self.file_dir = "/files/"
self.ds = DataStorage("sqlite:///"+os.path.dirname(os.path.abspath(__file__)) + self.file_dir+"cache.db")
print "Current Cache size: " + str(self.cache_folder_size(
os.path.dirname(os.path.abspath(__file__)) + self.file_dir)) + " MB"
class DataFetcher(object):
def __init__(self,):
super(DataFetcher, self).__init__()
self.file_dir = "/files/"
self.ds = DataStorage("sqlite:///"+os.path.dirname(os.path.abspath(__file__)) + self.file_dir+"cache.db")
print "Current Cache size: " + str(self.cache_folder_size(
os.path.dirname(os.path.abspath(__file__)) + self.file_dir)) + " MB"
def fetch_data(self, url_list):
list = {}
for url in url_list:
list[url] = self.get_file(url)
return list
def cache_folder_size(self, folder):
folder_size = 0
for (path, dirs, files) in os.walk(folder):
for file in files:
filename = os.path.join(path, file)
folder_size += os.path.getsize(filename)
return folder_size / 1000000
def get_file(self, url):
#check if in chache
data_dict = self.ds.retrieve_dict(url)
headers = self.check_if_valid_file_last_update(url)
if data_dict is not None and headers is not None:
# get the f*****g date from the header
if datetime_utils.parse_date(headers['Last-Modified']) > datetime_utils.parse_date(data_dict['time']):
self.download(url)
for path in data_dict['path']:
os.remove(path)
self.ds.remove_dict(url)
else:
return data_dict['path']
else:
return self.download(url)
def generate_file_path(self):
return os.path.dirname(os.path.abspath(__file__)) + self.file_dir + str(uuid1())
def unzip(self, path, dirname, url):
if not os.path.exists(dirname):
os.makedirs(dirname)
zfile = zipfile.ZipFile(path)
for name in zfile.namelist():
zfile.extract(name, dirname)
zip_files = os.listdir(dirname)
results = []
for zip_file in zip_files:
path = self.generate_file_path()
os.rename(dirname +'/'+ zip_file, path)
results.append(path)
return results
def get_unzip_list(self, filename, type, url, data_dict):
if type.find("zip") != -1:
list_zips = self.unzip(filename, os.path.dirname(os.path.abspath(__file__)) + self.file_dir + 'zips', url)
os.remove(data_dict['path'])
self.ds.remove_dict(url)
save_dict = dict(path=list_zips, time=data_dict['time'])
self.ds.store_dict(url, save_dict)
return list_zips
else:
return None
def check_if_valid_file_last_update(self, url):
try:
tag_filter = ["image", "text/html"]
response = urllib2.urlopen(HeadRequest(url)).info()
found = False
for tag in tag_filter:
if response['Content-Type'].find(tag) != -1:
found = True
if not found:
return response
else:
return None
except:
return None
def download(self, url):
headers = self.check_if_valid_file_last_update(url)
if headers is not None:
path = self.generate_file_path()
save_dict = dict(path=path, time=headers['Last-Modified'])
urllib.urlretrieve(url, path)
unzip_result = self.get_unzip_list(path, headers['Content-Type'], url, save_dict)
if unzip_result is None:
self.ds.store_dict(url, save_dict)
return [path]
else:
return unzip_result
else:
return None
def test(self):
print self.fetch_data(
["http://www.pagepersonnel.it/index.html", "http://upload.wikimedia.org/wikipedia/it/3/30/Ls_xterm.png",
"http://gis.csi.it/repertorio/sitad_wgs84/DBPR10_ELEMIDRI/elemidri.zip", "http://www.google.it"])
print "second_dictionary", self.ds.retrieve_dict("hhh")
def __init__(self):
self.url_manager = UrlManager()
self.html_downloader = HtmlDownloader()
self.html_parser = HtmlParser()
self.data_storage = DataStorage()
def __init__(self, host, port, level):
DataStorage.__init__(self, level)
self._storage = StrictRedis(host=host, port=port)
def __init__(self, level):
self._storage = {}
DataStorage.__init__(self, level)