def method(self, *args):
args = list(args)
args.insert(0, attribute)
connection = Client(self.address, authkey=self.authkey)
connection.send(dumps(args))
result = loads(connection.recv())
connection.close()
if result[0] == 'RESULT':
return result[1]
else:
raise MemoClientException(result[1])
def pop(self, identifier=None):
self.lock.acquire()
if identifier:
path = os.path.join(self.path(), identifier)
if not os.path.exists(path):
return None
else:
identifier = os.listdir(self.path())[0]
path = os.path.join(self.path(), identifier)
with open(path, 'rb') as f:
data = loads(f.read())
os.remove(path)
self.lock.release()
return data
def load(cls, txn, identifier):
# try to load from transaction if it exists
if identifier in txn.elements:
return txn.elements[identifier]
# we cannot load from cache since it might not be a good version
# of the element
path = os.path.join(txn.database.path(), cls.directory(), identifier)
if not os.path.exists(path):
raise ElementNotFound(cls, identifier)
# max_element will be the elected element, if at the end of the
# election algorithm max_element is still min_element then the
# requested element does not exist
version = choice(os.listdir(path))
# initial direction is ascend because we want the highest
# version
while True:
if version in txn.database.cache:
element = txn.database.cache[version].copy()
else:
with open(os.path.join(path, version), 'rb') as f:
value = loads(f.read())
element = cls(txn, identifier, version, value)
# Since we still don't know if the element will be edited
# we can only load a valid element for the transaction which
# means the latest element with a valid ts. A valid ts means
# that the associated transaction is commited and wasn't started
# at the beggining of this transaction.
if txn.is_commited(element.creation_ts()):
expired_at = element.expire_ts()
if expired_at and txn.is_commited(expired_at):
version = element.next
if version:
continue
else:
raise ElementNotFound(cls, identifier)
else:
break
else:
version = element.previous
if version:
continue
else:
raise ElementNotFound(cls, identifier)
# if the cache has already the element it will increase its hit count
txn.database.cache[element.version] = element
txn.elements[element.identifier] = element
return element
def main():
_setproctitle('structurarium.taskqueue')
parser = argparse.ArgumentParser(
description='Run Structurarium graph server'
)
parser.add_argument(
'--version',
'-v',
action='version',
version=__version__
)
parser.add_argument('host')
parser.add_argument('port', type=int)
parser.add_argument('path')
parser.add_argument('--authkey', '-k')
parser.add_argument(
'--worker',
'-w',
action='store',
type=int,
help='default is set to the number of CPU'
)
args = parser.parse_args()
listener = Listener((args.host, args.port), family='AF_INET')
database = TaskQueue(args.path, authkey=args.authkey)
print 'Running on %s:%s' % (args.host, args.port)
if args.worker > 1:
pool = Pool(processes=args.worker)
while True:
pool = Pool(processes=args.worker)
connection = listener.accept()
connection = cPickle.dumps(reduce_connection(connection))
database.process(connection)
pool.apply_async(process, [database, connection])
else:
print 'monothread'
database.replay()
while True:
connection = listener.accept()
command = loads(connection.recv())
output = database.play(command)
connection.send(dumps(output))
connection.close()
def load(cls, txn, identifier):
"""Try to load an element of this class based on an identifier. This
method *must* register the loaded instance to the transaction.
If the element is not found, an :class:`ElementNotFound` exception is
raised.
"""
# try to load from transaction if it exists
key = cls.key(identifier)
if key in txn.elements:
return txn.elements.get(key)
# try to load from cache
if key in txn.database.cache:
return txn.database.cache[key]
path = os.path.join(txn.database.path(), cls.directory(), identifier)
if not os.path.exists(path):
raise ElementNotFound
with open(path, 'rb') as f:
value = loads(f.read())
element = cls(txn, identifier, value)
txn.database.cache[key] = element
txn.elements[key] = element
return element
def __iter__(self):
with open(self.path(), 'rb') as f:
for line in f:
yield loads(line)
def send_and_recv(self, *args):
connection = Client(self.address, authkey=self.authkey)
connection.send(dumps(args))
result = loads(connection.recv())
connection.close()
return result
