def __init__(self, shard_server,shard_port,shard_db,shard_gridfs_collection): self.style = color.color_style() try: client = MongoClient(shard_server,shard_port) self.db = client[shard_db] self.fs = gridfs.GridFS(self.db,shard_gridfs_collection) except Exception as e: print self.style.ERROR("ERROR(MongodbBookFilesStore): %s"%(str(e),)) traceback.print_exc()
def __init__(self,shard_server,shard_port,shard_db,shard_gridfs_collection,download_func=None): self.style = color.color_style() ##########from MediaPipeline########### self.spiderinfo = {} self.download_func = download_func ##########from MediaPipeline########### self.store = self._get_store(shard_server,shard_port,shard_db,shard_gridfs_collection) self.item_download = {}
def __init__(self, shard_server, shard_port, shard_db, shard_gridfs_collection): self.style = color.color_style() try: client = MongoClient(shard_server, shard_port) self.db = client[shard_db] self.fs = gridfs.GridFS(self.db, shard_gridfs_collection) except Exception as e: print self.style.ERROR("ERROR(MongodbBookFilesStore): %s" % (str(e), )) traceback.print_exc()
def __init__(self): self.style = color.color_style() try: client = pymongo.MongoClient(self.MONGODB_SERVER, self.MONGODB_PORT) self.db = client[self.MONGODB_DB] except Exception as e: print( self.style.ERROR("ERROR(ShardMongodbPipeline): %s" % (str(e), ))) traceback.print_exc()
def __init__(self): """ The only async framework that PyMongo fully supports is Gevent. Currently there is no great way to use PyMongo in conjunction with Tornado or Twisted. PyMongo provides built-in connection pooling, so some of the benefits of those frameworks can be achieved just by writing multi-threaded code that shares a MongoClient. """ self.style = color.color_style() try: client = MongoClient(self.MONGODB_SERVER,self.MONGODB_PORT) self.db = client[self.MONGODB_DB] except Exception as e: print self.style.ERROR("ERROR(ShardMongodbPipeline): %s"%(str(e),)) traceback.print_exc()
def __init__(self, shard_server, shard_port, shard_db, shard_gridfs_collection, download_func=None): self.style = color.color_style() ##########from MediaPipeline########### # self.spiderinfo = {} self.download_func = download_func ##########from MediaPipeline########### self.store = self._get_store(shard_server, shard_port, shard_db, shard_gridfs_collection) self.item_download = {}
def __init__(self, original_url="", *args): self.original_url = original_url self.style = color.color_style() DropItem.__init__(self, *args)
def __init__(self, file_url=None, *args): self.file_url = file_url self.style = color.color_style() Exception.__init__(self, *args)
def __init__(self, store_uri, download_func=None): if not store_uri: raise NotConfigured self.store = self._get_store(store_uri) self.style = color.color_style() super(FilePipeline, self).__init__(download_func=download_func)
def __init__(self): self.style = color.color_style()
def __init__(self,store_uri,download_func=None): if not store_uri: raise NotConfigured self.store = self._get_store(store_uri) self.style = color.color_style() super(FilePipeline, self).__init__(download_func=download_func)
def __init__(self, host="127.0.0.1", port=2003): self.style = color.color_style() self._sock = socket() self._sock.connect((host, port))
def __init__(self, host="127.0.0.1", port=2003): self.style = color.color_style() self._sock = socket() self._sock.connect((host,port))