def __init__(self, namespace, redis_host='127.0.0.1', redis_port=6379, redis_db=1, csv_file_path="/tmp/daodao.csv"): BasePipeline.__init__(self, namespace) self._out_file = open(csv_file_path, "wb") self._csv_writer = csv.writer(self._out_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL, lineterminator='\n') # 写标题拦 self._csv_writer.writerow([ "Name", "BLon", "BLat", "PlaySpend", "PlaySpendUnit", "Address", "Tel", "OpenTime", "Rating", "TicketInfo", "ZipCode", "Description", "Comments", "CommentsNumber", "SortSequence" ]) try: temp_namespace = "%s:%s" % (namespace, "temp") self.temp_item_dict = RedisDict(temp_namespace, host=redis_host, port=redis_port, db=redis_db) except RedisError, e: self.logger.error("redis error %s" % e)
def __init__(self, namespace, redis_host="127.0.0.1", redis_port=6379, redis_db=1, csv_file_path="/tmp/daodao.csv"): BasePipeline.__init__(self, namespace) self._out_file = open(csv_file_path, "wb") self._csv_writer = csv.writer( self._out_file, delimiter=",", quotechar='"', quoting=csv.QUOTE_ALL, lineterminator="\n" ) # 写标题拦 self._csv_writer.writerow( [ "Name", "BLon", "BLat", "PlaySpend", "PlaySpendUnit", "Address", "Tel", "OpenTime", "Rating", "TicketInfo", "ZipCode", "Description", "Comments", "CommentsNumber", "SortSequence", ] ) try: temp_namespace = "%s:%s" % (namespace, "temp") self.temp_item_dict = RedisDict(temp_namespace, host=redis_host, port=redis_port, db=redis_db) except RedisError, e: self.logger.error("redis error %s" % e)
def __init__(self, namespace, redis_host='192.168.11.108', redis_port=6379, redis_db=0, db_host='192.168.11.195', db_port=5432, db_user='******', db_password='******', db_base='test'): BasePipeline.__init__(self, namespace) try: redis_namespace = "%s:%s" % (namespace, "temp") self._temp_redis_dict = RedisDict(redis_namespace, host=redis_host, port=redis_port, db=redis_db) self._db = DB(host=db_host, port=db_port, user=db_user, password=db_password, database=db_base) except RedisError, e: self.logger.error("init redis dict failed error:%s" % e) raise e
def __init__(self, namespace, redis_host="127.0.0.1", redis_port=6379, redis_db=1): BasePipeline.__init__(self, namespace) try: temp_namespace = "%s:%s" % (namespace, "temp") self.temp_item_dict = RedisDict(temp_namespace, host=redis_host, port=redis_port, db=redis_db) except RedisError, e: self.logger.error("redis error %s" % e)
def __init__(self, namespace): """init method Args: namespace: str, namespace """ BasePipeline.__init__(self, namespace) self.logger.debug("init city item pipeline")
def __init__(self, namespace, csv_file_path="/tmp/tag.csv"): BasePipeline.__init__(self, namespace) self._out_file = open(csv_file_path, "wb") self._csv_writer = csv.writer(self._out_file, delimiter=',', quotechar='"', quoting=csv.QUOTE_ALL, lineterminator='\n') self.logger.info("init tag item pipeline")
def __init__(self, namesapce, redis_host='192.168.11.108', redis_port=6379, redis_db=0): BasePipeline.__init__(self, namesapce) self.logger.info("init activity item pipline finished") try: redis_namespace = "%s:%s" % (namesapce, "temp") self.temp_item_dict = RedisDict(redis_namespace, host=redis_host, port=redis_port, db=redis_db) except RedisError, e: self.logger.error("redis error %s" % e) raise e
def __init__(self, namespace, db_host="127.0.0.1", db_port=5432, db_user="******", db_password="******", db_base="swift"): BasePipeline.__init__(self, namespace) self.logger.debug("init image item pipeline") try: self.item_db = DB(host=db_host, port=db_port, user=db_user, password=db_password, database=db_base) except DBError, e: self.logger.error("db error %s" % e)
def __init__(self, namespace, db_host="127.0.0.1", db_port=5432, db_user="******", db_password="******", db_base="swift"): """初始化DealItemPipeline 主要是链接数据库 """ BasePipeline.__init__(self, namespace) self.logger.debug("init nuomi DealItemPipeline") try: self.item_db = DB(host=db_host, port=db_port, user=db_user, password=db_password, database=db_base) except DBError, e: self.logger.error("db error %s" % e)
def __init__(self, namespace, redis_host='127.0.0.1', redis_port=6379, redis_db=1): BasePipeline.__init__(self, namespace) try: temp_namespace = "%s:%s" % (namespace, "temp") self.temp_item_dict = RedisDict(temp_namespace, host=redis_host, port=redis_port, db=redis_db) except RedisError, e: self.logger.error("redis error %s" % e)
def __init__(self, namespace, db_host="127.0.0.1", db_port=5432, db_user="******", db_password="******", db_base="swift"): """init method Args: ignore """ BasePipeline.__init__(self, namespace) self.logger.debug("init hotel code item pipeline") try: self.item_db = DB(host=db_host, port=db_port, user=db_user, password=db_password, database=db_base) except DBError, e: self.logger.error("db error %s" % e)
def __init__(self, namespace): BasePipeline.__init__(self, namespace) self.logger.debug("init 228com PictureItemPipeline")
def __init__(self, namespace, out_path=u"webitem.csv"): BasePipeline.__init__(self, namespace) self._out_file = open(out_path, u"wb") self._csv_file = csv.writer(self._out_file) self.logger.info(u"init WebItemPipeline finish")