def __init__(self,
namespace,
redis_host='127.0.0.1',
redis_port=6379,
redis_db=1,
csv_file_path="/tmp/daodao.csv"):
BasePipeline.__init__(self, namespace)
self._out_file = open(csv_file_path, "wb")
self._csv_writer = csv.writer(self._out_file,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_ALL,
lineterminator='\n')
# 写标题拦
self._csv_writer.writerow([
"Name", "BLon", "BLat", "PlaySpend", "PlaySpendUnit", "Address",
"Tel", "OpenTime", "Rating", "TicketInfo", "ZipCode",
"Description", "Comments", "CommentsNumber", "SortSequence"
])
try:
temp_namespace = "%s:%s" % (namespace, "temp")
self.temp_item_dict = RedisDict(temp_namespace,
host=redis_host,
port=redis_port,
db=redis_db)
except RedisError, e:
self.logger.error("redis error %s" % e)
def __init__(self, namespace, redis_host="127.0.0.1", redis_port=6379, redis_db=1, csv_file_path="/tmp/daodao.csv"):
BasePipeline.__init__(self, namespace)
self._out_file = open(csv_file_path, "wb")
self._csv_writer = csv.writer(
self._out_file, delimiter=",", quotechar='"', quoting=csv.QUOTE_ALL, lineterminator="\n"
)
# 写标题拦
self._csv_writer.writerow(
[
"Name",
"BLon",
"BLat",
"PlaySpend",
"PlaySpendUnit",
"Address",
"Tel",
"OpenTime",
"Rating",
"TicketInfo",
"ZipCode",
"Description",
"Comments",
"CommentsNumber",
"SortSequence",
]
)
try:
temp_namespace = "%s:%s" % (namespace, "temp")
self.temp_item_dict = RedisDict(temp_namespace, host=redis_host, port=redis_port, db=redis_db)
except RedisError, e:
self.logger.error("redis error %s" % e)
def __init__(self,
namespace,
redis_host='192.168.11.108',
redis_port=6379,
redis_db=0,
db_host='192.168.11.195',
db_port=5432,
db_user='******',
db_password='******',
db_base='test'):
BasePipeline.__init__(self, namespace)
try:
redis_namespace = "%s:%s" % (namespace, "temp")
self._temp_redis_dict = RedisDict(redis_namespace,
host=redis_host,
port=redis_port,
db=redis_db)
self._db = DB(host=db_host,
port=db_port,
user=db_user,
password=db_password,
database=db_base)
except RedisError, e:
self.logger.error("init redis dict failed error:%s" % e)
raise e
def __init__(self, namespace, redis_host="127.0.0.1", redis_port=6379, redis_db=1):
BasePipeline.__init__(self, namespace)
try:
temp_namespace = "%s:%s" % (namespace, "temp")
self.temp_item_dict = RedisDict(temp_namespace, host=redis_host, port=redis_port, db=redis_db)
except RedisError, e:
self.logger.error("redis error %s" % e)
def __init__(self, namespace):
"""init method
Args:
namespace: str, namespace
"""
BasePipeline.__init__(self, namespace)
self.logger.debug("init city item pipeline")
def __init__(self, namespace):
"""init method
Args:
namespace: str, namespace
"""
BasePipeline.__init__(self, namespace)
self.logger.debug("init city item pipeline")
def __init__(self, namespace, csv_file_path="/tmp/tag.csv"):
BasePipeline.__init__(self, namespace)
self._out_file = open(csv_file_path, "wb")
self._csv_writer = csv.writer(self._out_file,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_ALL,
lineterminator='\n')
self.logger.info("init tag item pipeline")
def __init__(self, namesapce, redis_host='192.168.11.108', redis_port=6379, redis_db=0):
BasePipeline.__init__(self, namesapce)
self.logger.info("init activity item pipline finished")
try:
redis_namespace = "%s:%s" % (namesapce, "temp")
self.temp_item_dict = RedisDict(redis_namespace, host=redis_host,
port=redis_port, db=redis_db)
except RedisError, e:
self.logger.error("redis error %s" % e)
raise e
def __init__(self, namespace, db_host="127.0.0.1",
db_port=5432, db_user="******",
db_password="******", db_base="swift"):
BasePipeline.__init__(self, namespace)
self.logger.debug("init image item pipeline")
try:
self.item_db = DB(host=db_host, port=db_port, user=db_user,
password=db_password, database=db_base)
except DBError, e:
self.logger.error("db error %s" % e)
def __init__(self, namespace, redis_host='192.168.11.108', redis_port=6379,
redis_db=0, db_host='192.168.11.195', db_port=5432, db_user='******',
db_password='******', db_base='test'):
BasePipeline.__init__(self, namespace)
try:
redis_namespace = "%s:%s" % (namespace, "temp")
self._temp_redis_dict = RedisDict(redis_namespace, host=redis_host, port=redis_port, db=redis_db)
self._db = DB(host=db_host, port=db_port, user=db_user, password=db_password, database=db_base)
except RedisError, e:
self.logger.error("init redis dict failed error:%s" % e)
raise e
def __init__(self, namespace, db_host="127.0.0.1", db_port=5432,
db_user="******", db_password="******", db_base="swift"):
"""初始化DealItemPipeline
主要是链接数据库
"""
BasePipeline.__init__(self, namespace)
self.logger.debug("init nuomi DealItemPipeline")
try:
self.item_db = DB(host=db_host, port=db_port, user=db_user,
password=db_password, database=db_base)
except DBError, e:
self.logger.error("db error %s" % e)
def __init__(self,
namespace,
redis_host='127.0.0.1',
redis_port=6379,
redis_db=1):
BasePipeline.__init__(self, namespace)
try:
temp_namespace = "%s:%s" % (namespace, "temp")
self.temp_item_dict = RedisDict(temp_namespace,
host=redis_host,
port=redis_port,
db=redis_db)
except RedisError, e:
self.logger.error("redis error %s" % e)
def __init__(self,
namesapce,
redis_host='192.168.11.108',
redis_port=6379,
redis_db=0):
BasePipeline.__init__(self, namesapce)
self.logger.info("init activity item pipline finished")
try:
redis_namespace = "%s:%s" % (namesapce, "temp")
self.temp_item_dict = RedisDict(redis_namespace,
host=redis_host,
port=redis_port,
db=redis_db)
except RedisError, e:
self.logger.error("redis error %s" % e)
raise e
def __init__(self,
namespace,
db_host="127.0.0.1",
db_port=5432,
db_user="******",
db_password="******",
db_base="swift"):
"""init method
Args:
ignore
"""
BasePipeline.__init__(self, namespace)
self.logger.debug("init hotel code item pipeline")
try:
self.item_db = DB(host=db_host,
port=db_port,
user=db_user,
password=db_password,
database=db_base)
except DBError, e:
self.logger.error("db error %s" % e)
def __init__(self, namespace):
BasePipeline.__init__(self, namespace)
self.logger.debug("init 228com PictureItemPipeline")
def __init__(self, namespace, out_path=u"webitem.csv"):
BasePipeline.__init__(self, namespace)
self._out_file = open(out_path, u"wb")
self._csv_file = csv.writer(self._out_file)
self.logger.info(u"init WebItemPipeline finish")
def __init__(self, namespace, out_path=u"webitem.csv"):
BasePipeline.__init__(self, namespace)
self._out_file = open(out_path, u"wb")
self._csv_file = csv.writer(self._out_file)
self.logger.info(u"init WebItemPipeline finish")
def __init__(self, namespace):
BasePipeline.__init__(self, namespace)
self.logger.debug("init 228com PictureItemPipeline")
def __init__(self, namespace, csv_file_path="/tmp/tag.csv"):
BasePipeline.__init__(self, namespace)
self._out_file = open(csv_file_path, "wb")
self._csv_writer = csv.writer(self._out_file, delimiter=',', quotechar='"',
quoting=csv.QUOTE_ALL, lineterminator='\n')
self.logger.info("init tag item pipeline")
