def test_parse_order_valid(self):
"""If the given order is valid, return an Order namedtuple with the correct values."""
shove = Shove({})
order = shove.parse_order(
'{"project": "asdf", "command": "qwer", "log_key": 77, '
'"log_queue": "zxcv"}')
eq_(order.project, 'asdf')
eq_(order.command, 'qwer')
eq_(order.log_key, 77)
eq_(order.log_queue, 'zxcv')
def test_update(self):
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.update(tstore)
self.assertEqual(self.store['min'], 6)
def __init__(self, tokens_filename, audio, log):
self._log = log
self._audio = audio
self._tokens_filename = tokens_filename
self._eventQueue = queue.Queue()
persist_path = "/tmp"
for directory in ("alerts", "alerts/all", "alerts/active"):
d = os.path.join(persist_path, directory)
if not os.path.exists(d):
os.mkdir(d)
# would prefer to use sqlite, but that complains about
# our threads accessing the same connection - and dbm seems to not
# store any changes.
self.allAlerts = Shove("file:///tmp/alerts/all")
self.activeAlerts = Shove("file:///tmp/alerts/active")
#print(list(self.allAlerts.values()))
self._last_user_activity = datetime.datetime.now()
t = threading.Thread(target=self.eventQueueThread, daemon=True)
t.start()
GObject.timeout_add(500, self.alertCheck)
def get_data_set_attribute(self, data_set, key):
attrib_name = self.get_attribute_name(data_set, key)
if self.has_attribute(data_set, key):
return self.store[attrib_name]
else:
return None
self.store.close()
gc.collect()
self.store = Shove('file://' + self.filename,
'memory://',
optimize=False)
def test_execute_invalid_command(self):
"""If the given command could not be found for the given project, return an error tuple."""
shove = Shove({'myproject': path('test_project')})
order = Order(project='myproject',
command='foo',
log_key=5,
log_queue='asdf')
procfile_path = path('test_project', 'bin', 'commands.procfile')
eq_(shove.execute(order),
(1, 'No command `foo` found in {0}'.format(procfile_path)))
def getCacheInstance(server='dirCache'):
try:
from shove import Shove
print 'sqlite:///%s.sqlite' % server
dirCache = Shove('sqlite:///%s.sqlite' % server)
#ftpCache = Shove()
print 'use shove'
except:
dirCache = {}
print 'use dict'
return dirCache
def test_update(self):
from shove import Shove
tstore = Shove()
tstore['max'] = 3
tstore['min'] = 6
tstore['pow'] = 7
self.store['max'] = 2
self.store['min'] = 3
self.store['pow'] = 7
self.store.update(tstore)
self.store.sync()
self.assertEqual(self.store['min'], 6)
def __init__(self, rank, world_size, comm, stats_dir=''):
self.rank = rank
self.world_size = world_size
self.comm = comm
self.send = self.comm.send
self.recv = self.comm.recv
# TODO: Make cleaner.
if stats_dir == None:
stats_dir = ''
else:
if stats_dir[-1] != '/':
stats_dir = stats_dir + '/'
self.initial_pos = GameState(GameState.INITIAL_POS)
self.root = self.initial_pos.get_hash(self.world_size)
self.work = PriorityQueue()
folder_path = 'file://' + stats_dir + 'stats/' + str(self.rank) + '/'
self.resolved = Shove(folder_path + 'results')
self.remote = Shove(folder_path + 'remote')
# As for recieving, should test them when appropriate
# in the run loop.
self.received = []
# Keep a dictionary of "distributed tasks"
# Should contain an id associated with the length of task.
# For example, you distributed rank 0 has 4, you wish to
# distribute 3, 2. Give it an id, like 1 and associate it
# with length 2. Then once all the results have been received
# you can compare the length, and then reduce the results.
# solving this particular distributed task.
self._id = 0 # Job id tracker.
self._counter = {} # A job_id -> Number of results
# remaining.
self._pending = {} # job_id -> [ Job, GameStates, ... ]
# Resolved.
self.stats_dict = Shove(folder_path +
'stats') # Statistics for process.
self.stats_dict["num_lookups"] = 0
def __init__(self,
src_file,
title='',
version=None,
use_binary=True,
inputPositionAnnotationName="protein_change",
outputPositionAnnotationName="aapos"):
super(PositionTransformingDatasource, self).__init__(src_file,
title=title,
version=version)
self.db = Shove(src_file, "memory://")
self.inputAnnotationName = inputPositionAnnotationName
self.outputAnnotationName = self.title + "_" + outputPositionAnnotationName
self.proteinRegexp = re.compile("[A-Z\*a-z]*([0-9]+)")
def __init__(self, data_dir, batch_size, seq_length):
self.data_dir = data_dir
root = Shove('file://' + data_dir)
self.batch_size = batch_size
self.seq_length = seq_length
tensor_file = os.path.join(data_dir, "data.npy")
if not os.path.exists(tensor_file):
print "reading text file"
self.preprocess(root, tensor_file)
else:
print "loading preprocessed files"
self.load_preprocessed(tensor_file)
self.create_batches()
self.reset_batch_pointer()
def poll(self):
stats_cache = Shove(config.get('stats_cache'))
wiki = Wiki()
try:
user = config.get('fedoracommunity.connector.fas.'
'minimal_user_name')
passwd = config.get('fedoracommunity.connector.fas.'
'minimal_user_password')
except KeyError:
pass
if user and passwd:
try:
wiki.login(user, passwd)
self.log.info('Logging into wiki as user %s' % user)
except AuthError, e:
self.log.info('Wiki login failed: %s' % e)
def __init__(self, access_token=None, session=None):
self.expires_at = None
self.session = session or MemoryStorage()
if isinstance(session, six.string_types):
from shove import Shove
from wechatpy.session.shovestorage import ShoveStorage
querystring = get_querystring(session)
prefix = querystring.get('prefix', ['wechatpy'])[0]
shove = Shove(session)
storage = ShoveStorage(shove, prefix)
self.session = storage
self.session.set('access_token', access_token)
def test_process_order_valid(self):
"""If parse_order returns a valid order, execute it and send logs back to Captain."""
shove = Shove({})
order = Order(project='asdf',
command='qwer',
log_key=23,
log_queue='zxcv')
shove.parse_order = Mock(return_value=order)
shove.execute = Mock(return_value=(0, 'output'))
eq_(shove.process_order('{"project": "asdf"}'),
('zxcv',
JSON({
'version': '1.0',
'log_key': 23,
'return_code': 0,
'output': 'output'
})))
shove.execute.assert_called_with(order)
def flot_wiki_edits_per_day(self, **params):
stats_cache = Shove(config.get('stats_cache'))
try:
data = stats_cache['wiki_all_revisions']
except KeyError:
return False
timestamps = defaultdict(int)
for rev_id in data['revs']:
timestamp = data['revs'][rev_id]['time']
day_timestamp = int(mktime((timestamp[0], timestamp[1],
timestamp[2], 0, 0, 0, 0, 0, 0))*1000)
timestamps[day_timestamp] += 1
flot = {'data': [], 'options': {'xaxis': {'mode': 'time'}}}
timestamps_sorted = timestamps.keys()
timestamps_sorted.sort()
flot['data'] = [[[timestamp, timestamps[timestamp]] \
for timestamp in timestamps_sorted][:-1]]
return flot
def __init__(self, appid, access_token=None, session=None, timeout=None, auto_retry=True):
self.appid = appid
self.expires_at = None
self.session = session or MemoryStorage()
self.timeout = timeout
self.auto_retry = auto_retry
if isinstance(session, six.string_types):
from shove import Shove
from wechatpy.session.shovestorage import ShoveStorage
querystring = get_querystring(session)
prefix = querystring.get('prefix', ['wechatpy'])[0]
shove = Shove(session)
storage = ShoveStorage(shove, prefix)
self.session = storage
if access_token:
self.session.set(self.access_token_key, access_token)
def __init__(self, component_appid, component_appsecret, session=None):
"""
:param component_appid: 第三方平台appid
:param component_appsecret: 第三方平台appsecret
:param component_verify_ticket: 微信后台推送的ticket，此ticket会定时推送
"""
self.component_appid = component_appid
self.component_appsecret = component_appsecret
self.expires_at = None
self.session = session or MemoryStorage()
if isinstance(session, six.string_types):
from shove import Shove
from wechatpy.session.shovestorage import ShoveStorage
querystring = get_querystring(session)
prefix = querystring.get('prefix', ['wechatpy'])[0]
shove = Shove(session)
storage = ShoveStorage(shove, prefix)
self.session = storage
def test_execute_valid_order(self):
shove = Shove({'myproject': path('test_project')})
order = Order(project='myproject',
command='pwd',
log_key=5,
log_queue='asdf')
with patch('shove.base.Popen') as Popen:
p = Popen.return_value
p.communicate.return_value = 'command output', None
p.returncode = 0
return_code, output = shove.execute(order)
Popen.assert_called_with(['pwd'],
cwd=path('test_project'),
stdout=PIPE,
stderr=STDOUT)
p.communicate.assert_called_with()
eq_(return_code, 0)
eq_(output, 'command output')
import wikipedia as pywikibot
import getopt
try:
from hashlib import md5
except ImportError: # Python 2.4 compatibility
from md5 import new as md5
import os
import re
import subprocess
import sys
import time
import urllib
import urllib2
import os
from shove import Shove
file_store = Shove('file://mystore')
def saveName(title):
name = urllib.unquote(title)
file_store[name] = title
def isNewTitle(name):
name = urllib.unquote(name)
try :
if (file_store[name] ) :
print "Skipping %s" % name
return 0
else:
return 1
except KeyError :
print "not seen %s" % name
def setUp(self):
from shove import Shove
self.store = Shove('hdf5://test.hdf5/test')
def test__cmp__(self):
from shove import Shove
tstore = Shove()
self.store['max'] = 3
tstore['max'] = 3
self.assertEqual(self.store, tstore)
def setUp(self):
self.store = Shove(self.s3string, compress=True)
def setUp(self):
self.store = Shove('file://test', compress=True)
def setUp(self):
self.store = Shove('simple://', compress=True)
#!/usr/bin/env python
from __future__ import print_function
import sys
from shove import Shove
import cStringIO
import re
import os
import stat
import multiprocessing
import threading
from collections import defaultdict
# global stuff
shove = Shove('sqlite:///dump.db')
print(type(shove))
print(shove)
from Article import Article
from Title import Title
from clean_text import clean_text
with open('config.json') as json_data_file:
config = json.load(json_data_file)
config_shove = config.get("shove")
config_db = config.get("wiki_db")
config_categories = config.get("categories")
shove_buffer = config_shove.get("buffer", 10)
shove_folder = config_shove.get("folder_path", "./")
dataset = config.get("train_dataset")
if config.get("verbose", False):
log.basicConfig(format="%(message)s", level=log.INFO)
articles_dict = Shove("file://" + shove_folder + "/articles",
sync=shove_buffer)
titles_dict = Shove("file://" + shove_folder + "/titles", sync=shove_buffer)
words_dict = Shove("file://" + shove_folder + "/words", sync=shove_buffer)
categories_dict = {}
cf_w_dict = {}
R_w_dict = Shove("file://" + shove_folder + "/R_w", sync=shove_buffer)
vocabulary_c_dict = Shove("file://" + shove_folder + "/vocabulary_c",
sync=shove_buffer)
lang = 'en'
p_nonword = re.compile(r'\W')
def process_articles():
pages = 0
def __init__(self, path):
self.jobs = []
self.path = path
self.store = Shove(path, optimize=False)
def setUp(self):
from shove import Shove
self.store = Shove(self.svnstring, compress=True)
DMS_NEAT_FIELDS = set(['center_dec', 'predicted_dec'])
HMS_NEAT_FIELDS = set(['center_ra', 'predicted_ra'])
URL_BASE = 'http://skyview.gsfc.nasa.gov/'
IMAGE_SIZE = 500 # pixels
# The html is so bad that BeautifulSoup won't work, so we parse with a regex
IMAGE_PARSING_REGEX = re.compile("img src='(.*?)'")
##### Redis Config
REDIS_PREFIX = 'skymorph'
redis = StrictRedis(host='localhost', port=6379, db=3)
##### Disk cache config
store = Shove('file:///var/asterank/skymorph_store',
'file:///var/asterank/skymorph_cache')
store_mutex = Lock()
##### Functions
def images_for(target, n=10):
results = search_target(target)
threads = []
ret = []
for result in results[-1 * 10:]:
ret.append({
'key': result['key'],
'time': result['time'],
})
t = Thread(target=get_image, args=(result['key'], ))
# -*- coding: utf-8 -*-
__version__ = '$Id: speedydeletion.py 9692 2011-10-30 15:03:29Z xqt $'
#
import re, sys, string
import wikipedia as pywikibot
from pywikibot import i18n
import config, pagegenerators, catlib
import replace
import xmlreader
from shove import Shove
file_store = Shove('file://wikiaupload')
import signal
import sys
def signal_handler(signal, frame):
print 'You pressed Ctrl+C!'
sys.exit(0)
signal.signal(signal.SIGINT, signal_handler)
import unicodedata
def decode(link):
b = link
link = unicode(link, 'utf-8')
link = unicodedata.normalize('NFKD', link)
from twisted.web import client
from twisted.web.client import HTTPPageGetter, HTTPClientFactory
from twisted.internet import reactor, protocol, defer
from datetime import timedelta
from feedcache import Cache
from shove import Shove
from tg import config
from moksha.hub import MokshaHub
from moksha.api.streams import PollingDataStream
log = logging.getLogger('moksha.hub')
feed_storage = Shove(config['feed_cache'], compress=True)
feed_cache = Cache(feed_storage)
class ConditionalHTTPPageGetter(HTTPPageGetter):
def handleStatus_200(self):
""" Attempt to save the last-modified header """
if self.headers.has_key('last-modified'):
self.factory.lastModified(self.headers['last-modified'])
def handleStatus_304(self):
""" Close the connection """
self.factory.notModified()
self.transport.loseConnection()
