def get_checksum(self):
"""
Calculates a checksum of the data in a predictable way.
"""
m = md5.new()
m.update(helpers.dump_json(self.data))
return m.hexdigest()
def _pull(self):
# Ping the instance and get its version
_logger.info( '%s Ping', self.address)
# _logger.info( "dump_json: %s", sync.cluster_state.get_state() + [{'address': sync.cluster_state.me}])
# Try to push your state to the other side
helpers.push_state( self.address, helpers.dump_json( sync.cluster_state.get_state() + [{'address': sync.cluster_state.me}]))
info = helpers.info( self.address)
if not info:
self._reachable = False
return
try:
self._version = data.DataVersion.from_dict( info[ 'version'])
except (TypeError, KeyError):
_logger.error( '%s Invalid pulled data', self.address)
return
self._touch_last_reachable()
# Check that the other instance has newer configuration
if self._version <= data.cur_data().version:
return
# The instance has newer configuration, try to pull it
_logger.info( '%s Pull', self.address)
content = helpers.pull( self.address)
if content is None:
return False
# Check in new data
if data.push_data( content):
config.save_configuration()
def get_checksum( self): """ Calculates a checksum of the data in a predictable way. """ m = md5.new() m.update( helpers.dump_json( self.data)) return m.hexdigest()
def main():
f_article = "articles"
f_filtered= "filtered"
f_final = "final.json"
args = parse_arguments()
# scrape end store NBA recaps
if args.scrape:
articles = get_site_text(args.date, args.days)
save_pickle(articles, f_article)
# load scraped articles
articles = load_pickle(f_article)
filtered = []
for art in articles:
starts = []
print(art[0])
records = get_records(art[1], starts)
streaks = get_streaks(art[1], starts)
# cleaning headers like TIP-INS
records = filter_out_upper(records)
streaks = filter_out_upper(streaks)
filtered.append((art[0], streaks, records))
# save filtered text to a pickle file
save_pickle(filtered, f_filtered)
filtered = load_pickle(f_filtered)
extracted = get_extracted_articles(filtered)
finals = []
for art in extracted:
final = dict()
final['score'] = art[0]
final['rec'] = art[2]
final['streaks'] = art[1]
finals.append(final)
dump_json(finals, f_final)
def main():
data = helpers.load_json("data/states.json")
if not isinstance(data, dict):
data = { x["full_name"]:x for x in data }
key = "marriage_age"
new = {}
lines = helpers.read_lines("entry.txt")
lines = [ x for x in lines if x ]
# lines = lines[::4]
for line in lines:
line = line.split(". ")[-1]
name, num = line.split(": ", 1)
new[name] = float(num)
try:
name = line.split("\t")[0]
name = name.split("(")[0].strip()
new[name] = float(line.split("\t")[1].replace(",", ""))
except Exception:
pass
[ print(k, ":", v) for k, v in new.items() ]
for name, val in new.items():
if name not in data:
data[name] = {}
data[name][key] = val
# Clean up the data
cleaned = {}
for k, v in data.items():
key = rmchars(k, ".")
key = key.replace("Saint", "St")
if key in cleaned:
cleaned[key].update(v)
else:
cleaned[key] = v
cleaned[key]["name"] = key
return helpers.dump_json(cleaned, "foo.json")
def save_configuration():
global _data_dir
# Don't write if there's no destination
if _data_dir is None:
return False
rm_old_files()
snapshot = {}
# Get a raw copy of all data
snapshot[ 'copy'] = data.get_copy()
# Get current system state
snapshot[ 'cluster'] = sync.cluster_state.get_state()
# Write this configuration
file_name = _data_dir + '/' +helpers.now().strftime( DATA_DIR_STRFTIME)
with open(file_name, 'w') as f:
f.write( helpers.dump_json( snapshot))
return True
def save_configuration():
global _data_dir
# Don't write if there's no destination
if _data_dir is None:
return False
rm_old_files()
snapshot = {}
# Get a raw copy of all data
snapshot['copy'] = data.get_copy()
# Get current system state
snapshot['cluster'] = sync.cluster_state.get_state()
# Write this configuration
file_name = _data_dir + '/' + helpers.now().strftime(DATA_DIR_STRFTIME)
with open(file_name, 'w') as f:
f.write(helpers.dump_json(snapshot))
return True
def _push(self):
"""Tries to push the current data to the other instance. """
# Check that current configuration is newer than that on the other instance
xdata = data.cur_data()
if self._version >= xdata.version:
return
# Push data
_logger.info( '%s Push', self.address)
result = helpers.push( self.address, helpers.dump_json({
'version': {
'sequence': xdata.version.sequence,
'checksum': xdata.version.checksum,
},
'data': xdata.data,
}))
_logger.info('push result: %s', result)
# Mark time when we tried to push new data
if result:
self._touch_last_push()
def _push(self):
"""Tries to push the current data to the other instance. """
# Check that current configuration is newer than that on the other instance
xdata = data.cur_data()
if self._version >= xdata.version:
return
# Push data
_logger.info('%s Push', self.address)
result = helpers.push(
self.address,
helpers.dump_json({
'version': {
'sequence': xdata.version.sequence,
'checksum': xdata.version.checksum,
},
'data': xdata.data,
}))
_logger.info('push result: %s', result)
# Mark time when we tried to push new data
if result:
self._touch_last_push()
def _pull(self):
# Ping the instance and get its version
_logger.info('%s Ping', self.address)
# _logger.info( "dump_json: %s", sync.cluster_state.get_state() + [{'address': sync.cluster_state.me}])
# Try to push your state to the other side
helpers.push_state(
self.address,
helpers.dump_json(sync.cluster_state.get_state() +
[{
'address': sync.cluster_state.me
}]))
info = helpers.info(self.address)
if not info:
self._reachable = False
return
try:
self._version = data.DataVersion.from_dict(info['version'])
except (TypeError, KeyError):
_logger.error('%s Invalid pulled data', self.address)
return
self._touch_last_reachable()
# Check that the other instance has newer configuration
if self._version <= data.cur_data().version:
return
# The instance has newer configuration, try to pull it
_logger.info('%s Pull', self.address)
content = helpers.pull(self.address)
if content is None:
return False
# Check in new data
if data.push_data(content):
config.save_configuration()
def _response_json( self, response): return self._response( 200, 'application/json', helpers.dump_json( response))
def call(self, data):
filename = value_or_callable(self.filename, data)
dump_json(filename, data, *self.args, **self.kwargs)
def _response_json(self, response):
return self._response(200, 'application/json',
helpers.dump_json(response))
