def get_checksum(self): """ Calculates a checksum of the data in a predictable way. """ m = md5.new() m.update(helpers.dump_json(self.data)) return m.hexdigest()
def _pull(self): # Ping the instance and get its version _logger.info( '%s Ping', self.address) # _logger.info( "dump_json: %s", sync.cluster_state.get_state() + [{'address': sync.cluster_state.me}]) # Try to push your state to the other side helpers.push_state( self.address, helpers.dump_json( sync.cluster_state.get_state() + [{'address': sync.cluster_state.me}])) info = helpers.info( self.address) if not info: self._reachable = False return try: self._version = data.DataVersion.from_dict( info[ 'version']) except (TypeError, KeyError): _logger.error( '%s Invalid pulled data', self.address) return self._touch_last_reachable() # Check that the other instance has newer configuration if self._version <= data.cur_data().version: return # The instance has newer configuration, try to pull it _logger.info( '%s Pull', self.address) content = helpers.pull( self.address) if content is None: return False # Check in new data if data.push_data( content): config.save_configuration()
def get_checksum( self): """ Calculates a checksum of the data in a predictable way. """ m = md5.new() m.update( helpers.dump_json( self.data)) return m.hexdigest()
def main(): f_article = "articles" f_filtered= "filtered" f_final = "final.json" args = parse_arguments() # scrape end store NBA recaps if args.scrape: articles = get_site_text(args.date, args.days) save_pickle(articles, f_article) # load scraped articles articles = load_pickle(f_article) filtered = [] for art in articles: starts = [] print(art[0]) records = get_records(art[1], starts) streaks = get_streaks(art[1], starts) # cleaning headers like TIP-INS records = filter_out_upper(records) streaks = filter_out_upper(streaks) filtered.append((art[0], streaks, records)) # save filtered text to a pickle file save_pickle(filtered, f_filtered) filtered = load_pickle(f_filtered) extracted = get_extracted_articles(filtered) finals = [] for art in extracted: final = dict() final['score'] = art[0] final['rec'] = art[2] final['streaks'] = art[1] finals.append(final) dump_json(finals, f_final)
def main(): data = helpers.load_json("data/states.json") if not isinstance(data, dict): data = { x["full_name"]:x for x in data } key = "marriage_age" new = {} lines = helpers.read_lines("entry.txt") lines = [ x for x in lines if x ] # lines = lines[::4] for line in lines: line = line.split(". ")[-1] name, num = line.split(": ", 1) new[name] = float(num) try: name = line.split("\t")[0] name = name.split("(")[0].strip() new[name] = float(line.split("\t")[1].replace(",", "")) except Exception: pass [ print(k, ":", v) for k, v in new.items() ] for name, val in new.items(): if name not in data: data[name] = {} data[name][key] = val # Clean up the data cleaned = {} for k, v in data.items(): key = rmchars(k, ".") key = key.replace("Saint", "St") if key in cleaned: cleaned[key].update(v) else: cleaned[key] = v cleaned[key]["name"] = key return helpers.dump_json(cleaned, "foo.json")
def save_configuration(): global _data_dir # Don't write if there's no destination if _data_dir is None: return False rm_old_files() snapshot = {} # Get a raw copy of all data snapshot[ 'copy'] = data.get_copy() # Get current system state snapshot[ 'cluster'] = sync.cluster_state.get_state() # Write this configuration file_name = _data_dir + '/' +helpers.now().strftime( DATA_DIR_STRFTIME) with open(file_name, 'w') as f: f.write( helpers.dump_json( snapshot)) return True
def save_configuration(): global _data_dir # Don't write if there's no destination if _data_dir is None: return False rm_old_files() snapshot = {} # Get a raw copy of all data snapshot['copy'] = data.get_copy() # Get current system state snapshot['cluster'] = sync.cluster_state.get_state() # Write this configuration file_name = _data_dir + '/' + helpers.now().strftime(DATA_DIR_STRFTIME) with open(file_name, 'w') as f: f.write(helpers.dump_json(snapshot)) return True
def _push(self): """Tries to push the current data to the other instance. """ # Check that current configuration is newer than that on the other instance xdata = data.cur_data() if self._version >= xdata.version: return # Push data _logger.info( '%s Push', self.address) result = helpers.push( self.address, helpers.dump_json({ 'version': { 'sequence': xdata.version.sequence, 'checksum': xdata.version.checksum, }, 'data': xdata.data, })) _logger.info('push result: %s', result) # Mark time when we tried to push new data if result: self._touch_last_push()
def _push(self): """Tries to push the current data to the other instance. """ # Check that current configuration is newer than that on the other instance xdata = data.cur_data() if self._version >= xdata.version: return # Push data _logger.info('%s Push', self.address) result = helpers.push( self.address, helpers.dump_json({ 'version': { 'sequence': xdata.version.sequence, 'checksum': xdata.version.checksum, }, 'data': xdata.data, })) _logger.info('push result: %s', result) # Mark time when we tried to push new data if result: self._touch_last_push()
def _pull(self): # Ping the instance and get its version _logger.info('%s Ping', self.address) # _logger.info( "dump_json: %s", sync.cluster_state.get_state() + [{'address': sync.cluster_state.me}]) # Try to push your state to the other side helpers.push_state( self.address, helpers.dump_json(sync.cluster_state.get_state() + [{ 'address': sync.cluster_state.me }])) info = helpers.info(self.address) if not info: self._reachable = False return try: self._version = data.DataVersion.from_dict(info['version']) except (TypeError, KeyError): _logger.error('%s Invalid pulled data', self.address) return self._touch_last_reachable() # Check that the other instance has newer configuration if self._version <= data.cur_data().version: return # The instance has newer configuration, try to pull it _logger.info('%s Pull', self.address) content = helpers.pull(self.address) if content is None: return False # Check in new data if data.push_data(content): config.save_configuration()
def _response_json( self, response): return self._response( 200, 'application/json', helpers.dump_json( response))
def call(self, data): filename = value_or_callable(self.filename, data) dump_json(filename, data, *self.args, **self.kwargs)
def _response_json(self, response): return self._response(200, 'application/json', helpers.dump_json(response))