def run(): sets = Pipeline(time.site_id, time.site_name).structure_set() Pipeline(time.site_id, time.site_name).open_spider(sets) for item in Time().first_requests(): Pipeline(time.site_id, time.site_name).process_item(item) Pipeline(time.site_id, time.site_name).upload_item(item, sets) try: Pipeline(time.site_id, time.site_name).close_spider() except: Logger().setLogger(time.log_path, 4, "Failed to close spider,db_session may failed")
def run(): sets = Pipeline(bbc.site_id, bbc.site_name).structure_set() Pipeline(bbc.site_id, bbc.site_name).open_spider(sets) urls = BBC().first_requests() for item in BBC().second_requests(urls): Pipeline(bbc.site_id, bbc.site_name).process_item(item) Pipeline(bbc.site_id, bbc.site_name).upload_item(item, sets) try: Pipeline(bbc.site_id, bbc.site_name).close_spider() except: Logger().setLogger(bbc.log_path, 4, "Failed to close spider,db_session may failed") pass
def __init__(s, dtype, stages, pipeq, bypassq): s.in_ = InValRdyBundle(dtype) s.out = OutValRdyBundle(dtype) s.in_q = InValRdyQueue(dtype, pipe=pipeq) s.out_q = OutValRdyQueue(dtype, bypass=bypassq) s.pipe = Pipeline(stages) s.connect(s.in_, s.in_q.in_) s.connect(s.out, s.out_q.out) @s.tick def logic(): # Automatically enq from input / deq from output s.in_q.xtick() s.out_q.xtick() # No stall if not s.out_q.is_full(): # Insert item into pipeline from input queue if not s.in_q.is_empty(): s.pipe.insert(s.in_q.deq()) # Items graduating from pipeline, add to output queue if s.pipe.ready(): s.out_q.enq(s.pipe.remove()) # Advance the pipeline s.pipe.advance()
def test_Pipeline(dump_vcd, stages): # Create the pipeline pipeline = Pipeline(stages) pipeline.vcd_file = dump_vcd # Fill up the pipeline i = -1 for i in range(stages - 1): pipeline.advance() pipeline.insert(i) assert not pipeline.ready() # Insert one last item pipeline.advance() pipeline.insert(i + 1) # Make sure there is something at the tail of the pipeline assert pipeline.ready() # Start removing items from the pipeline for i in range(stages): assert pipeline.ready() assert pipeline.remove() == i pipeline.advance() assert not pipeline.ready()
def run(): sets = Pipeline(hq.site_id, hq.site_name).structure_set() Pipeline(hq.site_id, hq.site_name).open_spider(sets) detail_url = Huanqiu().first_requests() for item in Huanqiu().second_requests(detail_url): Huanqiu().process_item(item) Pipeline(hq.site_id, hq.site_name).process_item(item) Pipeline(hq.site_id, hq.site_name).upload_item(item, sets) try: Pipeline(hq.site_id, hq.site_name).close_spider() except: Logger().setLogger(hq.log_path, 4, "Failed to close spider,db_session may failed") pass
def __init__( s, out, nstages=1 ): s.nstages = nstages # instantiate a single-entry bypass queue adapter s.out_q = OutValRdyQueueAdapter( out ) # instantiate a cycle-level pipeline if s.nstages > 0: s.pipe = Pipeline( s.nstages )
def __init__(self, browser, settings, *a, **kw): self.browser = browser self.settings = settings if self.browser: self._init_from_db() self.internal_err = True self.deny = True self.clawed_urls = [] self.site_urls = [] self.currentUrl = "" self.started = False self.pipeline = Pipeline(self.settings.get('MYSERVER_URI')) self.proxyUrl = self.settings.get("SPLASH_URL") logger.info('spider init is finished!')
def assess(model, df, columns, metrics, n_splits=5, early_stopping_rounds=20, verbose=0): """ k-fold cross-validation Checkpoints saving strategy ... :param model: sklearn-like object :param df: DataFrame with X and y :param columns: column names splited by types like utils.split_columns_by_types :param metrics: sklearn.metrics like function :param n_splits: the number of folds :param early_stopping_rounds: LightGBM param :param verbose: 0 - no logs, 1 - info, 2 - debug :return: iterations log """ if n_splits == 1: total_rows = df.shape[0] train_size = int(0.95 * total_rows) splits = [(df.index[:train_size], df.index[train_size:])] else: splits = kfold_with_respect_to_groups(df, n_splits=n_splits) log = [] for train_index, valid_index in splits: print('\n---------------------------') with Timer('Data Preparation:', verbose): pipeline = Pipeline(**columns, verbose=verbose) x_train = pipeline.fit_transform(df.loc[train_index, :]) y_train = df.loc[train_index, columns['target']] x_valid = pipeline.transform(df.loc[valid_index, :]) y_valid = df.loc[valid_index, columns['target']] with Timer('Fitting:', verbose): model.fit( x_train, y_train, eval_set=[(x_valid, y_valid)], early_stopping_rounds=early_stopping_rounds, verbose=-1 if verbose != 2 else 1, ) # with Timer('Postprocessing:', verbose): # pred_train = scores_postprocessing( # df=df.loc[train_index, :], # predicted=model.predict(x_train), # columns=columns, # is_test=False, # )[columns['target']] # pred_valid = scores_postprocessing( # df=df.loc[valid_index, :], # predicted=model.predict(x_valid), # columns=columns, # is_test=False, # )[columns['target']] pred_train, pred_valid = model.predict(x_train), model.predict( x_valid) with Timer('Saving:', verbose): train_score = metrics(y_train, pred_train) valid_score = metrics(y_valid, pred_valid) step = dict( model=model, pipeline=pipeline, train_score=train_score, valid_score=valid_score, not_adj_train_score=metrics(y_train, model.predict(x_train)), not_adj_valid_score=metrics(y_valid, model.predict(x_valid)), train_index=train_index, valid_index=valid_index, path=None, cached=False, ) try: step = save_model(step) except Exception: if verbose == 1: print("Warning: Couldn't save the model") log.append(step) gc.collect() if verbose == 1: print(step['train_score'], step['valid_score']) print('---------------------------\n') if verbose == 1: print('Erasing cache ...') for idx, step in enumerate( sorted(log, key=lambda dct: dct['valid_score'], reverse=True)): if idx == 0: step['best'] = True continue step['best'] = False try: os.remove(step['path']) if verbose == 2: print('Removed:', step['abspath']) except Exception: if verbose == 2: print("Warning: Couldn't remove file:", step['abspath']) return log