def test():
config = {
'ftp': {
'type': 1,
'host': '61.147.114.73',
'port': 21,
'user': '******',
'passwd': '123456x',
'timeout': 5,
'local': './'
}
}
analysis_engine = AnalysisEngine(config)
file_list = analysis_engine.input_data('~/text_storage/60006bak')
i = 0
count = len(file_list)
while i < count:
tlist = file_list[i:i + 4]
i += 5
start_time = time.time()
for t in tlist:
analysis_engine.process_file_data(60006, '~/text_storage/60006bak',
t, 0)
end_time = time.time()
mlog.log().info("analysis file count %d expend %d", i,
end_time - start_time)
def test_local():
config = {
'local': {
'type': 3,
'path': '/Users/kerry/work/pj/gitfork/mcrawler/client'
}
}
analysis_engine = AnalysisEngine(config)
analysis_engine.process_file_data(60007, './', 'xueqiu.db', 2)
def run(console):
analysis_engine = AnalysisEngine()
file_list = analysis_engine.input_data(console.path)
i = 0
count = len(file_list)
while i < count:
tlist = file_list[i:i + 4]
i += 5
start_time = time.time()
for t in tlist:
analysis_engine.process_file_data(console.plt_id, console.path, t,
0)
end_time = time.time()
mlog.log().info("analysis file count %d expend %d", i,
end_time - start_time)
def parser_xueqiu():
mconfig = {
'local': {
'type': 3,
'path': '/Users/kerry/work/pj/gitfork/mcrawler'
},
'result': {
'60006': {
'type': 5,
'name': '../discuss.db'
},
}
}
ae = AnalysisEngine(mconfig)
ae.process_file_data(60006, './file/', 'xueqiu.db', 2, -600)
"""
def parser_ftp_method(config, path, pid):
ae = AnalysisEngine(config)
ae.start()
file_list = ae.input_data(path)
i = 0
count = len(file_list)
while i < count:
unit_list = file_list[i:i + 5]
i += 5
start_time = time.time()
for t in unit_list:
ae.process_file_data(pid, path, t, 0)
end_time = time.time()
mlog.log().info("analysis file count %d expend %d", i,
end_time - start_time)
'host': '61.147.114.85:9092,61.147.114.80:9092,61.147.114.81:9092',
'name': 'newsparser_task_algo'
},
'result': {
'60006': {
'type': 5,
'name': '../discuss.db'
},
'60008': {
'type': 5,
'name': '../hexunstock.db'
}
}
}
analysis_engine = AnalysisEngine(config)
def tcallback(data):
analysis_engine.process_file_data(int(data.get('attr_id')),
data.get('key_name'),
data.get('pos_name'), 0)
def realtime():
analysis_engine.start(tcallback)
if __name__ == '__main__':
if platform.system() == "Darwin" or platform.system() == "Linux":
reload(sys)
def parser_local_method(config, path, name, pid, tid):
ae = AnalysisEngine(config)
ae.process_file_data(pid, path, name, 2, tid)