Python process_data 예제들

예제 #1

파일: search_combine.py 프로젝트: basketfish/Python

__author__ = 'Yukai Tseng'
import searcher
import data_load
import indexer

d=indexer.process_data("raw_data.pickle","fortunes_shelve")
searcher.search("fortunes_shelve")

예제 #2

파일: search_combine.py 프로젝트: ashokvardhanreddy/python

import searcher
import indexer

d= indexer.process_data("raw_data.txt","fortunes_shelve")
d=indexer.process_data("raw_data1.txt","fortunes_shelve")

searcher.search("fortunes_shelve")

예제 #3

파일: search_combine.py 프로젝트: HNO22/Python

import searcher1
import data_load
import indexer

d= indexer.process_data("raw_data.pickle")
#print(d)
searcher1.search(d)

예제 #4

파일: search_combine.py 프로젝트: snan1/hw-3-file-traverser

import searcher
import indexer
import data_load
dict_words = indexer.process_data("raw_data.txt","shelve_file")
searcher.search("shelve_file")

예제 #5

파일: Homework7-SearchEngine.py 프로젝트: bmarshallpf/Python

returnList = []
file_data = get_traversal_data()
web_data = visit_url(seed, "www.newhaven.edu", returnList)

webPickle = "raw_web.pickle"
dataPickle = "raw_data.pickle"

# catch exceptions dealing with pickling the objects, as well as catching 
# any exceptions dealing with opening the file to begin with
try:
	with open(webPickle, "bw") as out:
		try:
			pickle.dump(web_data, out)
		except pickle.PicklingError:
			print("Unpicklable object passed into dump().")
except IOError as ioe:
	print("Unable to write to file: " + ioe.filename)

# catch exceptions dealing with pickling the objects, as well as catching 
# any exceptions dealing with opening the file to begin with
try:
	with open(dataPickle, "bw") as out:
		try:
			pickle.dump(file_data, out)
		except pickle.PicklingError:
			print("Unpicklable object passed into dump().")
except IOError as ioe:
	print("Unable to write to file: " + ioe.filename)

indexer.process_data("unh_shelve", "indexed_files", dataPickle, webPickle)
searcher.search("unh_shelve", "indexed_files")

예제 #6

파일: search_combine.py 프로젝트: snan1/hw4-web-crawler

import searcher
import indexer
from indexer import query
import data_load
import crawler_new

indexer.process_data("raw_data.txt","shelve_file")
indexer.process_data("raw_data1.txt","shelve_file")

searcher.search("shelve_file",query)

예제 #7

파일: search_combine.py 프로젝트: HNO22/Python

import searcher1
import data_load
import indexer

d = indexer.process_data("raw_data.pickle", "crawler1.pickle")
#print(d)
searcher1.search(d)

예제 #8

	except URLError as e:
		print("error")

	return returnList

from data_load import get_traversal_data
import indexer
import searcher
import pickle

crawler_backlog = {}
seed = "http://www.newhaven.edu/"
crawler_backlog[seed]=0
returnList = []
file_data = get_traversal_data()
web_data = visit_url(seed, "www.newhaven.edu", returnList)

webPickle = "raw_web.pickle"
dataPickle = "raw_data.pickle"

out = open(webPickle, "bw")
pickle.dump(web_data, out)
out.close()

out = open(dataPickle, "bw")
pickle.dump(file_data, out)
out.close()

indexer.process_data("unh_shelve", "indexed_files", dataPickle, webPickle)
searcher.search("unh_shelve", "indexed_files")

예제 #9

파일: search_combine.py 프로젝트: justicewang/Python

import indexer
import data_load
import searcher

fliename = get_traversal_data()  # bulid file pickle and return filename
d = indexer.process_data(fliename)  # use pickle bulid DB file and return the db name

print(searcher.searching(d))

예제 #10

import data_load
import searcher
import indexer
import webcrawler

data_load.traverser()
webcrawler.webcrawler()
d = indexer.process_data("raw_data.pickle", "webdata.pickle")

searcher.search("fortune_shelve")

예제 #11

파일: search_combine.py 프로젝트: rmeji1/Python

import searcher
import indexer

d = indexer.process_data("raw_data.pickle", "fortune_shelves");
d = indexer.process_data("urls.pickle", "fortune_shelves");
searcher.search("fortune_shelves");

예제 #12

import data_load
import searcher
import indexer


data_load.traverser()
d=indexer.process_data("raw_data.pickle","fortunes_shelve")
searcher.search("fortunes_shelve")

예제 #13

파일: Homework4.py 프로젝트: bmarshallpf/Python

#Brandon Marshall       
#Python Scripting
#October 1, 2015
#Homework 4 - File Traverser

import data_load
import indexer
import searcher

data_load.get_traversal_data()
indexer.process_data("raw_data.pickle", "fortunes_shelve", "indexed_files")
searcher.search("fortunes_shelve", "indexed_files")

예제 #14

파일: search_combine.py 프로젝트: HNO22/Python

import searcher1
import data_load
import indexer

d= indexer.process_data("raw_data.pickle","crawler1.pickle")
#print(d)
searcher1.search(d)

예제 #15

#Brandon Marshall
#Python Scripting
#October 1, 2015
#Homework 4 - File Traverser

import data_load
import indexer
import searcher

data_load.get_traversal_data()
indexer.process_data("raw_data.pickle", "fortunes_shelve", "indexed_files")
searcher.search("fortunes_shelve", "indexed_files")