def save_templates(fn, site_id, templates): tpls = {} for i,p in ((x.page_id, page_to_dict(x)) for x in templates): if i not in tpls: tpls[i] = [] tpls[i].append(p) try: with open(fn, 'r') as f: obj = json.load(f) except IOError: obj = { 'templates': [], 'sites': {}, } if 'sites' not in obj or not obj['sites'] or not isinstance(obj['sites'], dict): obj['sites'] = {} obj['sites'].update( [( site_id, list(tpls.keys()) )] ) for i,p in ( (x['page_id'],x) for x in obj['templates'] ): if i not in tpls: tpls[i] = [] tpls[i].append(p) obj['templates'] = list(chain(*tpls.values())) with open(fn, 'w') as f: json.dump(obj, f)
def save_templates(fn, site_id, templates): tpls = {} for i, p in ((x.page_id, page_to_dict(x)) for x in templates): if i not in tpls: tpls[i] = [] tpls[i].append(p) try: with open(fn, 'r') as f: obj = json.load(f) except IOError: obj = { 'templates': [], 'sites': {}, } if 'sites' not in obj or not obj['sites'] or not isinstance( obj['sites'], dict): obj['sites'] = {} obj['sites'].update([(site_id, list(tpls.keys()))]) for i, p in ((x['page_id'], x) for x in obj['templates']): if i not in tpls: tpls[i] = [] tpls[i].append(p) obj['templates'] = list(chain(*tpls.values())) with open(fn, 'w') as f: json.dump(obj, f)
def tofile(self, file): """Store the scraper into the given file-like object""" tpls = [page_to_dict(x) for x in self._templates] json.dump({'templates': tpls}, file)
def _save_templates(self, templates): with open(self.filename, 'w') as f: templates = [page_to_dict(t) for t in templates] return json.dump({'templates': templates}, f)