class NewsItem:
def __init__(self, json):
self.id = json['id']
self.type = json['type']
self.title = json['title']
self.description = json['description']
self.publish_time = datetime.datetime.strptime(json['published_at'], '%Y-%m-%dT%H:%M:%S+%f')
self.last_modification_time = datetime.datetime.strptime(json['modified_at'], '%Y-%m-%dT%H:%M:%S+%f')
self.interesting_counter_lock = Lock()
self.interesting_counter = 0
if 'image' in json:
self.main_image_url = json['image']['formats'][0]['url']['jpg']
self.content = []
self.categories = []
for i in json['categories']:
self.categories.append(NewsCategory(i))
self._parse_content_json_children(json['content']['children'])
def mark_interesting(self):
yield from self.interesting_counter_lock
self.interesting_counter += 1
self.interesting_counter_lock.release()
def _parse_content_json_children(self, json):
for child in json:
childType = child['type']
if childType == 'text' or childType == 'title':
self.content.append(NewsItemContent(child))
elif childType == 'container':
self._parse_content_json_children(child['children'])
elif childType == 'external_content':
self._read_external_content(child['external_content'])
elif childType == 'link_container':
pass # Do nothing really
elif childType == 'video' or childType == 'image' or childType == 'quote' or childType == 'audio' or childType == 'carousel':
pass # print(childType, " is not currently supported")
else:
print("Title:", self.title)
print(json)
print('Unhandled content type ', childType)
return
def _read_external_content(self, json):
content_type = json['content_type']
if content_type == 'twitter':
self.content.append(NewsItemContent({'type': 'tweet', 'url': json['url']}))
elif content_type == 'youtube':
self.content.append(NewsItemContent({'type': 'youtube', 'url': json['url']}))
else:
print('External content type', content_type, 'not currently supported')
print(json)
def __init__(self,
maxsize: int = 0,
order_fn: Callable[[TTask], Any] = identity,
*,
loop: AbstractEventLoop = None) -> None:
self._maxsize = maxsize
self._full_lock = Lock(loop=loop)
self._open_queue = PriorityQueue(maxsize, loop=loop)
self._task_wrapper = SortableTask.orderable_by_func(order_fn)
self._id_generator = count()
self._tasks = set()
self._in_progress = {}
def __init__(self, bot):
super().__init__(bot)
self.claim_lock = Lock()
self.release_lock = Lock()
self.claimed = dict()
self.channel_reclaimer.start()
self.claimed_category = bot.get_channel(CLAIMED_CATEGORY_ID)
self.pool_category = bot.get_channel(POOL_CATEGORY_ID)
self.open_category = bot.get_channel(OPEN_CATEGORY_ID)
def __init__(self, bot):
self.bot = bot
self.feeds = self.load_feeds()
self.feedLock = Lock()
self.bot.RSSaddInternal = self.addInternal
self.bot.RSSremoveInternal = self.removeInternal
self.bot.RSSlistInternal = self.listInternal
"""
Of the format:
{User: {url : {ignores:set, contain:set, read:list(Item)}}}
"""
ensure_future(self.check())
class Limit:
sleep_time: SleepTime = field(default_factory=SleepTime)
def __post_init__(self):
self.lock = Lock()
async def trigger(self):
async def func():
await sleep(self.sleep_time.get())
self.lock.release()
await self.lock.acquire()
create_task(func())
def __init__(self, world, app_name, screen_size):
self.__world = world
self.__screen_size = screen_size
self.__logger = logging.getLogger(app_name)
self.__game = PyGameManager(app_name, screen_size)
self.vehicle = None
self.camera = None
self.collision_sensor = None
self.collision = False
self.__actors = []
self.__logger.info("Create environment...")
self.__init()
self.__colliding = Lock()
async def synced_task(name, cycles: int, shared_lock: asyncio.Lock):
for i in range(cycles):
print("%s - jdu charapt" % name)
await asyncio.sleep(random.random() * 2)
print("%s - cekam na lock" % name)
await shared_lock.acquire()
print("%s - mam lock a makam" % name)
await asyncio.sleep(random.random() * 2)
print("%s - hotovo, vracim lock" % name)
shared_lock.release()
print("%s - lock vracen" % name)
print("Hotovo, mrdam na to, jdu domu, zasarana prace.")
async def process_and_upload_data(path: str, get_file_data: FileFetcherType,
container: str, base_path: str) -> NoReturn:
"""
Uploads processed files to the storage using the correct
caching and ``content-type`` specs.
Parameters
----------
path: str
Path (within the storage container) in which the
file is to be stored.
get_file_data: FileFetcherType
base_path: str
container: str
Storage container in which the file is to be stored.
Returns
-------
NoReturn
"""
_, file_name = split_path(path)
# Files are stored as JSON - the extension must be updated:
file_name, _ = splitext(file_name)
json_name = f"{file_name}.json"
yaml_name = f"{file_name}.yaml"
json_path = str.join(processor_settings.URL_SEPARATOR,
[STORAGE_PATH, json_name])
yaml_path = str.join(processor_settings.URL_SEPARATOR,
[STORAGE_PATH, yaml_name])
if ".github" in path:
return None
raw_data = await get_file_data(path, base_path)
data = await prepare_data(raw_data)
# Uploading the data
with StorageClient(container=container, path=json_path) as client:
async with Lock():
client.upload(data=data.json_data)
with StorageClient(container=container,
path=yaml_path,
content_type="application/x-yaml") as client:
async with Lock():
client.upload(data=data.yaml_data)
class UpdateManager(object):
def __init__(self, name=None):
self.lock = Lock()
self.name = name or random_key(6)
self.loss_history = []
self.n_updates = 0
self._reset_state()
def _reset_state(self):
self.update_name = "update_{}_{:05d}".format(self.name, self.n_updates)
self.clients = set()
self.client_responses = dict()
self.update_meta = None
@property
def in_progress(self):
return self.lock.locked()
@property
def clients_left(self):
return len(self.clients) - len(self.client_responses)
def __len__(self):
return self.in_progress * len(self.clients)
async def start_update(self, **update_meta):
print("starting update")
if self.in_progress:
raise UpdateInProgress
self._reset_state()
await self.lock.acquire()
self.update_meta = update_meta
def end_update(self):
self.lock.release()
self.n_updates += 1
return self.client_responses
def client_start(self, client_id):
if not self.in_progress:
raise UpdateNotInProgress
self.clients.add(client_id)
def client_end(self, client_id, response):
if not self.in_progress:
raise UpdateNotInProgress
self.client_responses[client_id] = response
print("Update finished: {} [{}/{}]".format(client_id,
len(self.client_responses),
len(self.clients)))
def __init__(self,
loop: asyncio.AbstractEventLoop,
host='https://api.coinpaprika.com',
version='v1'):
self.loop = loop
self.host = host
self.lock = Lock(loop=loop)
self.version = version
self.request_counter = 0
self.request_second = 0
self.headers = {
'Accept': 'application/json',
'Accept-Charset': 'utf-8'
}
async def archive(self, record: ArchiveRecord):
"""Process a record to write to storage.
Acquire a path lock before archive. Writing to storage will only be
allowed to a path if a valid `path_lock` is held and the path is not
locked by another process.
Parameters
----------
record : ArchiveRecord
A notebook and where it should be written to storage
"""
async with self.path_lock_ready:
lock = self.path_locks.get(record.filepath)
if lock is None:
lock = Lock()
self.path_locks[record.filepath] = lock
# Skip writes when a given path is already locked
if lock.locked():
self.log.info("Skipping archive of %s", record.filepath)
return
async with lock:
try:
async with self.session.create_client(
's3',
aws_secret_access_key=self.settings.
s3_secret_access_key,
aws_access_key_id=self.settings.s3_access_key_id,
endpoint_url=self.settings.s3_endpoint_url,
region_name=self.settings.s3_region_name,
) as client:
self.log.info("Processing storage write of %s",
record.filepath)
file_key = s3_key(self.settings.workspace_prefix,
record.filepath)
await client.put_object(Bucket=self.settings.s3_bucket,
Key=file_key,
Body=record.content)
self.log.info("Done with storage write of %s",
record.filepath)
except Exception as e:
self.log.error('Error while archiving file: %s %s',
record.filepath,
e,
exc_info=True)
def __init__(self, filename: str):
if not isfile(filename):
with open(filename, 'wb') as file:
# indicator
file.write(OP)
# root
file.write(pack('Q', 9))
self.root = IndexNode(is_leaf=True)
self.root.dump(file)
else:
with open(filename, 'rb+') as file:
if file.read(1) == OP:
file.close()
return BasicEngine.repair(filename)
else:
ptr = unpack('Q', file.read(8))[0]
file.seek(ptr)
self.root = IndexNode(file=file)
file.seek(0)
file.write(OP)
self.allocator = Allocator()
self.async_file = AsyncFile(filename)
self.command_que = SortedList()
self.file = open(filename, 'rb+', buffering=0)
self.lock = Lock()
self.on_interval = (0, 1)
self.on_write = False
self.task_que = TaskQue()
def __init__(self, discovery, device_uid: str, device_ip: str) -> None:
"""Create a controller interface.
Usually this is called from the discovery service. If neither
device UID or address are specified, will search network for
exactly one controller. If UID is specified then the addr is
ignored.
Args:
device_uid: Controller UId as a string (eg: mine is '000013170')
If specified, will search the network for a matching device
device_addr: Device network address. Usually specified as IP
address
Raises:
ConnectionAbortedError: If id is not set and more than one iZone
instance is discovered on the network.
ConnectionRefusedError: If no iZone discovered, or no iZone
device discovered at the given IP address or UId
"""
self._ip = device_ip
self._discovery = discovery
self._device_uid = device_uid
self.zones = [] # type: List[Zone]
self.fan_modes = [] # type: List[Controller.Fan]
self._system_settings = {} # type: Controller.ControllerData
self._initialised = False
self._fail_exception = None
self._sending_lock = Lock()
async def __stop(self):
if self.__stop_lock is None:
self.__stop_lock = Lock()
async with self.__stop_lock:
if self.__running:
await wait_for(self.__stop_two_steps(),
timeout=self.graceful_shutdown_timeout)
def __init__(self):
config_filename = os.path.join(ROOT_DIR, "config", "config.yaml")
self.config = safe_load(open(config_filename, "r"))["timelord"]
self.free_servers: List[Tuple[str, str]] = list(
zip(self.config["vdf_server_ips"],
self.config["vdf_server_ports"]))
self.ips_estimate = {
k: v
for k, v in list(
zip(
self.config["servers_ips_estimate"]["ip"],
self.config["servers_ips_estimate"]["ips"],
))
}
self.lock: Lock = Lock()
self.server_count: int = len(self.free_servers)
self.active_discriminants: Dict[bytes32, Tuple[StreamWriter, uint64,
str]] = {}
self.best_weight_three_proofs: int = -1
self.active_discriminants_start_time: Dict = {}
self.pending_iters: Dict = {}
self.submitted_iters: Dict = {}
self.done_discriminants: List[bytes32] = []
self.proofs_to_write: List[OutboundMessage] = []
self.seen_discriminants: List[bytes32] = []
self.proof_count: Dict = {}
self.avg_ips: Dict = {}
self.discriminant_queue: List[Tuple[bytes32, uint64]] = []
self._is_shutdown = False
def __init__(self, config: NetworkConfig, loop: AbstractEventLoop):
self.config = config
self.sm = NetRomStateMachine(self, AsyncioTimer)
self.router = NetRomRoutingTable(config.node_alias())
self.l3_apps: Dict[AX25Call, str] = {}
# Mapping of destination addresses to protocol factory. When a new connection is made to the destination
# we will create a new instance of the protocol as well as a NetRom transport and add it to l3_connections
self.l3_servers: Dict[AX25Call, Callable[[], Protocol]] = {}
# This is a mapping of local circuit IDs to (transport, protocol). When incoming data is handled for a
# circuit, this is how we pass it on to the instance of the protocol
self.l3_connections: Dict[int, Tuple[Transport, Protocol]] = {}
# This is a mapping of local circuit ID to a protocol factory. These protocols do not yet have a transport and
# are in a half-opened state. Once a connect ack is received, a transport will be created and these will be
# migrated to l3_connections
self.l3_half_open: Dict[int, Callable[[], Protocol]] = {}
self.data_links: Dict[int, DataLinkManager] = {}
self.route_lock = Lock()
self.loop = loop
def extra():
return f"[L4 Call={str(config.node_call())} Alias={config.node_alias()}]"
LoggingMixin.__init__(self, logging.getLogger("main"), extra)
def __init__(self, devices: List[Device] = None) -> None:
self.__devices: List[Device] = devices or []
self.__position: int = None
self.__fix_position()
self.__token_lock: Lock = Lock()
if not self.__devices:
self.__devices.append(Device(__create_device_key()))
def __init__(self, server="localhost"):
handlers = {
MSG_READY_NOTIFY: self._ready_msg,
MSG_LSA_UPDATE_NOTIFY: self._lsa_change_msg,
MSG_LSA_DELETE_NOTIFY: self._lsa_change_msg,
MSG_NEW_IF: self._if_msg,
MSG_DEL_IF: self._if_msg,
MSG_ISM_CHANGE: self._if_change_msg,
MSG_NSM_CHANGE: self._nbr_change_msg,
MSG_REACHABLE_CHANGE: self._reachable_msg,
}
super().__init__(server, handlers)
self.ready_lock = Lock()
self.ready_cond = {
LSA_TYPE_OPAQUE_LINK: {},
LSA_TYPE_OPAQUE_AREA: {},
LSA_TYPE_OPAQUE_AS: {},
}
self.lsid_seq_num = {}
self.lsa_change_cb = None
self.opaque_change_cb = {}
self.reachable_routers = set()
self.reachable_change_cb = None
self.if_area = {}
self.ism_states = {}
self.ism_change_cb = None
self.nsm_states = {}
self.nsm_change_cb = None
def __init__(self,
interface: driver.I2CInterface,
callbacks_for_states: Mapping[driver.StateType,
Callable[[float], None]],
polling_interval: float = 0.01,
loop: AbstractEventLoop = None):
"""
Initialise a new Poller instance.
:param interface: The I2CInterface to use. Note that Poller assumes
exclusive ownership of the object and might operate on it from a
background thread.
:param callbacks_for_states: A map from hardware status update
driver.StateTypes to callbacks to invoke with the new value when
they occur.
:param polling_interval: The target interval between polling the
hardware for state updates, in seconds. The actual interval might
be longer if the event loop is busy.
:param loop: The event loop to use (None for asyncio default).
"""
self._interface = interface
self._callbacks_for_states = callbacks_for_states
self.polling_interval = polling_interval
self._loop = loop if loop else get_event_loop()
# Hardware communication lock.
self._interface_lock = Lock()
self._shutdown = False
# Start polling in a background coroutine.
self._poll_task = self._loop.create_task(self._run_poll_loop())
def __init__(self):
"""
Create new instance of FBaseProcessor that will process requests.
"""
self._processor_function_map = {}
self._annotations_map = {}
self._write_lock = Lock()
def main(args: Arguments) -> None:
logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
loop = asyncio.get_event_loop()
lock = Lock()
loop.run_until_complete(connect(args.token, args.ip, args.port, lock))
loop.close()
def __init__(self, trader):
super().__init__()
self.trader = trader
self.exchange = self.trader.get_exchange()
self.is_simulated = self.trader.simulate
self.side = None
self.symbol = None
self.origin_price = 0
self.origin_stop_price = 0
self.origin_quantity = 0
self.market_total_fees = 0
self.filled_quantity = 0
self.filled_price = 0
self.fee = None
self.currency, self.market = None, None
self.order_id = None
self.status = None
self.status = None
self.order_type = None
self.creation_time = 0
self.canceled_time = 0
self.executed_time = 0
self.last_prices = None
self.created_last_price = None
self.order_profitability = None
self.linked_to = None
self.is_from_this_octobot = True
self.linked_portfolio = None
self.order_notifier = None
self.linked_orders = []
self.lock = Lock()
def __init__(self,
intents,
token=None,
*,
shard_count=None,
shard_ids=None):
# Configurable stuff
self.intents = int(intents)
self.token = token
self.shard_count = shard_count
self.shard_ids = shard_ids
# Things used by the lib, usually doesn't need to get changed but
# can if you want to.
self.shards = []
self.loop = get_event_loop()
self.logger = getLogger("speedcord")
self.http = None
self.opcode_dispatcher = OpcodeDispatcher(self.loop)
self.event_dispatcher = EventDispatcher(self.loop)
self.gateway_handler = DefaultGatewayHandler(self)
self.connected = Event()
self.exit_event = Event(loop=self.loop)
self.remaining_connections = None
self.connection_lock = Lock(loop=self.loop)
# Default event handlers
self.opcode_dispatcher.register(0, self.handle_dispatch)
# Check types
if shard_count is None and shard_ids is not None:
raise TypeError("You have to set shard_count if you use shard_ids")
def __init__(
self,
username: str,
password: str,
*,
authentification_address: str,
avatar: int,
log_messages_in_console: bool,
server_address: str,
) -> None:
"""
Initialises interface.
"""
if authentification_address is None:
raise AttributeError(
"Unspecified authentification address. Please specify an authentification address."
)
self._authentification_address = authentification_address
self._avatar = avatar
self._log_messages_in_console = log_messages_in_console
self._logged_in = False
self._password = password
self._server_address = server_address
self._username = username
self._waiting_start = False
self._lock = Lock()
async def inner(self: Cog, ctx: Context, *args, **kwargs) -> None:
lock = func.__locks.setdefault(ctx.author.id, Lock())
if lock.locked():
embed = Embed()
embed.colour = Colour.red()
log.debug(f"User tried to invoke a locked command.")
embed.description = (
"You're already using this command. Please wait until it is done before you use it again."
)
embed.title = random.choice(ERROR_REPLIES)
await ctx.send(embed=embed)
return
async with func.__locks.setdefault(ctx.author.id, Lock()):
await func(self, ctx, *args, **kwargs)
def __init__(self,
service,
total_size,
chunk_size,
stream,
parallel,
encryptor=None,
padder=None,
**kwargs):
self.service = service
self.total_size = total_size
self.chunk_size = chunk_size
self.stream = stream
self.parallel = parallel
# Stream management
self.stream_start = stream.tell() if parallel else None
self.stream_lock = threading.Lock() if parallel else None
# Progress feedback
self.progress_total = 0
self.progress_lock = Lock() if parallel else None
# Encryption
self.encryptor = encryptor
self.padder = padder
self.response_headers = None
self.etag = None
self.last_modified = None
self.request_options = kwargs
def __init__(self, shard_id, client, loop: AbstractEventLoop):
self.id = shard_id
self.client = client
self.loop = loop
self.ws = None
self.gateway_url = None
self.logger = getLogger(f"speedcord.shard.{self.id}")
self.connected = Event(
loop=self.loop) # Some bots might wanna know which shards is
# online at all times
self.received_heartbeat_ack = True
self.heartbeat_interval = None
self.heartbeat_count = None
self.failed_heartbeats = 0
self.session_id = None
self.last_event_id = None # This gets modified by gateway.py
self.gateway_send_lock = Lock(loop=self.loop)
self.gateway_send_limit = 120
self.gateway_send_per = 60
self.gateway_send_left = self.gateway_send_limit
self.gateway_send_reset = time() + self.gateway_send_per
# Default events
self.client.opcode_dispatcher.register(10, self.handle_hello)
self.client.opcode_dispatcher.register(11, self.handle_heartbeat_ack)
self.client.opcode_dispatcher.register(9, self.handle_invalid_session)
self.client.event_dispatcher.register("READY", self.handle_ready)
def __init__(self, scribble_path: str):
self._scribble_path = join(BASE_DIR, scribble_path)
self._row = {
self._fields[i]: self._defaults[i]
for i in range(len(self._fields))
}
self._lock = Lock()
def __init__(self, proxy, manager):
if not isinstance(proxy, ParentProxy):
proxy = ParentProxy(proxy, proxy)
self.logger = logging.getLogger('hxs2')
self.proxy = proxy
self.name = self.proxy.name
self.timeout = 6 # read frame_data timeout
self._manager = manager
self._ping_test = False
self._ping_time = 0
self.connected = False
self.connection_lost = False
self.udp_relay_support = None
self.udp_event = None
self._psk = self.proxy.query.get('PSK', [''])[0]
self.method = self.proxy.query.get('method',
[DEFAULT_METHOD])[0].lower()
self.mode = int(self.proxy.query.get('mode', ['0'])[0])
self.hash_algo = self.proxy.query.get('hash',
[DEFAULT_HASH])[0].upper()
self.remote_reader = None
self.remote_writer = None
self._socport = None
self.__pskcipher = None
self.__cipher = None
self._next_stream_id = 1
self._settings_async_drain = None
self._client_writer = {}
self._client_status = {}
self._client_resume_reading = {}
self._client_drain_lock = {}
self._stream_status = {}
self._stream_addr = {}
self._stream_task = {}
self._last_active = {}
self._last_active_c = time.monotonic()
self._last_ping_log = 0
self._connection_task = None
self._connection_stat = None
self._last_direction = SEND
self._last_count = 0
self._buffer_size_ewma = 0
self._recv_tp_max = 0
self._recv_tp_ewma = 0
self._sent_tp_max = 0
self._sent_tp_ewma = 0
self._stat_data_recv = 0
self._stat_total_recv = 1
self._stat_recv_tp = 0
self._stat_data_sent = 0
self._stat_total_sent = 1
self._stat_sent_tp = 0
self._lock = Lock()
def __init__(self, trader):
self.trader = trader
self.exchange_manager = trader.exchange_manager
self.status = OrderStatus.OPEN
self.creation_time = time.time()
self.executed_time = 0
self.lock = Lock()
self.linked_orders = []
self.order_id = trader.parse_order_id(None)
self.simulated = trader.simulate
self.symbol = None
self.currency = None
self.market = None
self.taker_or_maker = None
self.timestamp = 0
self.origin_price = 0
self.created_last_price = 0
self.origin_quantity = 0
self.origin_stop_price = 0
self.order_type = None
self.side = None
self.filled_quantity = 0
self.linked_portfolio = None
self.linked_to = None
self.canceled_time = 0
self.fee = None
self.filled_price = 0
self.order_profitability = 0
self.total_cost = 0
# raw exchange order type, used to create order dict
self.exchange_order_type = None
def __init__(self, config: Dict, constants: Dict):
self.constants = constants
self.config: Dict = config
self.ips_estimate = {
k: v
for k, v in list(
zip(
self.config["vdf_clients"]["ip"],
self.config["vdf_clients"]["ips_estimate"],
))
}
self.lock: Lock = Lock()
self.active_discriminants: Dict[bytes32, Tuple[StreamWriter, uint64,
str]] = {}
self.best_weight_three_proofs: int = -1
self.active_discriminants_start_time: Dict = {}
self.pending_iters: Dict = {}
self.submitted_iters: Dict = {}
self.done_discriminants: List[bytes32] = []
self.proofs_to_write: List[OutboundMessage] = []
self.seen_discriminants: List[bytes32] = []
self.proof_count: Dict = {}
self.avg_ips: Dict = {}
self.discriminant_queue: List[Tuple[bytes32, uint128]] = []
self.max_connection_time = self.config["max_connection_time"]
self.potential_free_clients: List = []
self.free_clients: List[Tuple[str, StreamReader, StreamWriter]] = []
self._is_shutdown = False
self.server = None
async def process_head(filters: str, ordering: OrderingType,
arguments: QueryArguments,
date: str) -> QueryResponseType:
ordering_script = format_ordering(ordering)
query = DBQueries.exists.substitute(clause_script=filters,
ordering=await ordering_script)
logging.info(f"DB Query: {query}")
logging.info(f"Query arguments: {arguments}")
async with Lock():
items = container.query_items(query=query,
parameters=arguments,
max_item_count=MAX_ITEMS_PER_RESPONSE,
enable_cross_partition_query=True,
partition_key=date)
try:
results = list(items)
except KeyError:
raise NotAvailable()
if not len(results):
raise NotAvailable()
return list()
def __init__(self, max_age_seconds=120, loop=None):
assert max_age_seconds >= 1
self.store = {}
self.max_age = max_age_seconds
self.lock = Lock()
self.closed = False
self.loop = loop or get_event_loop()
self.clean_keys()
def __init__(self, channel):
self.channel = channel
self.loop = channel.loop
self.partial_message = None
self.callbacks = {}
self.expected_frame = None
self.expected_methods = ()
self.response = None
self.lock = Lock(loop=self.loop)
def __init__(self, inbox, outbox, loop=None):
self.inbox = inbox
self.outbox = outbox
if not loop:
loop = get_event_loop()
self._loop = loop
self._pause_lock = Lock(loop=self._loop)
self._stop_event = Event(loop=self._loop)
self._test = None
self.__testy = None
self.on_init()
def __init__(self, json):
self.id = json['id']
self.type = json['type']
self.title = json['title']
self.description = json['description']
self.publish_time = datetime.datetime.strptime(json['published_at'], '%Y-%m-%dT%H:%M:%S+%f')
self.last_modification_time = datetime.datetime.strptime(json['modified_at'], '%Y-%m-%dT%H:%M:%S+%f')
self.interesting_counter_lock = Lock()
self.interesting_counter = 0
if 'image' in json:
self.main_image_url = json['image']['formats'][0]['url']['jpg']
self.content = []
self.categories = []
for i in json['categories']:
self.categories.append(NewsCategory(i))
self._parse_content_json_children(json['content']['children'])
def __init__(self, target):
if not iscoroutinefunction(target):
raise TypeError("I find your lack of async disturbing.")
self.target = target #: target coroutine for this activity.
self.__trigger_obj = None
self.trigger_obj = trigger
self.mode = mode
self.lock = Lock()
# Create empty parameter dictionary
self.parameters = inspect.signature(target).parameters
self.empty_param_dict = {}
for param in self.parameters:
if param != "self":
self.empty_param_dict[param] = None
self._args = args
self._kwargs = kwargs
class Stock:
def __init__(self):
self.items = []
self.lock = Lock()
def get_display_items(self):
return [item for item in self.items if (item.state == ItemState.ON_DISPLAY)]
# requires lock fo state check and set
def fetch_item_from_display(self, item_type):
"""
Get an item from display, if available
:param item_type: the type of item requested
:return: The requested item or None
"""
item = None
yield from self.lock.acquire()
items = [item for item in self.items if (item.state == ItemState.ON_DISPLAY and item.type == item_type)]
if len(items) > 0:
item = items[0]
item.state = ItemState.IN_CART
self.lock.release()
yield item
# requires lock for id increment
def insert_item(self, item):
"""
Add an item to the warehouse
:param item: the item to add
:return: the item after added to the warehouse. Now uniquely identified
"""
assert item.type is not None
yield from self.lock.acquire()
item.id = len(self.items)
item.state = ItemState.ON_DISPLAY
self.items.append(item)
self.lock.release()
yield item
class FrameHandler(object):
"""Handler for incoming frames, also synchronizes the flow."""
def __init__(self, channel):
self.channel = channel
self.loop = channel.loop
self.partial_message = None
self.callbacks = {}
self.expected_frame = None
self.expected_methods = ()
self.response = None
self.lock = Lock(loop=self.loop)
def update(self, d):
self.callbacks.update(d)
@contextmanager
def wait_for(self, *expected):
"""Context manager used to synchronize the flow.
Example::
with handler.wait_for(QueueDeclareOK) as fut:
channel.queue_declare(*args)
yield from fut # Will lock until QueueDeclareOK is recieved
"""
yield from self.lock.acquire()
self.expected_frame = MethodFrame
self.expected_methods = expected
r = self.response = Future(loop=self.loop)
# Some methods may not wait for the reply
# In this case the code below will allow to instantly return
# when yielding from r
if expected[0] is None:
r.set_result(None)
try:
yield r
finally:
self.expected_frame = None
self.expected_methods = ()
self.response = None
self.lock.release()
def ready(self, result=None):
"""Notify synchronizer that the expected method was recieved."""
self.response.set_result(result)
def handle(self, frame, payload, *,
CONTENT_SPEC=(BasicReturn, BasicDeliver, BasicGetOK)):
"""Handle incoming frame"""
# At first, check if an unexpected frame was received
response = self.response
if self.expected_frame is not None and frame != self.expected_frame:
exc = UnexpectedFrame(UNEXPECTED_FRAME_FMT.format(
self.expected_frame, frame
))
response.set_exception(exc) # raise exception in the waiter
return
if frame == MethodFrame:
payload, method = frame.unpack(payload)
if method.synchronous:
# Check of an unexpected method was received
if not any(method == cls for cls in self.expected_methods):
exc = UnexpectedFrame(UNEXPECTED_FRAME_FMT.format(
self.expected_methods, method
))
# raise exception in the waiter
response.set_exception(exc)
return
if any(method == cls for cls in CONTENT_SPEC):
# Save what we have and wait for the message header
self.partial_message = BasicMessage()
self.expected_frame = HeaderFrame
else:
try:
callback = self.callbacks[method.method_type]
except KeyError:
# Recieved reply and there is no handler for it
if method.method_type not in METHODS:
response.set_exception(UnknownMethod(
UNKNOWN_METHOD_FMT.format(method.method_type)
))
self.ready(payload)
else:
# Schedule task to handle the method
async(callback(*payload))
elif frame == HeaderFrame:
msg = self.partial_message
msg._add_header(payload)
if msg._ready:
# Bodyless message
self.partial_message = None
response.set_result(msg)
else:
# Wait for the message body
self.expected_frame = BodyFrame
elif frame == BodyFrame:
msg = self.partial_message
msg._add_body_chunk(payload)
if msg._ready:
self.expected_frame = None
self.partial_message = None
response.set_result(msg)
class BasicEngine:
# 基础事务
def __init__(self, filename: str):
if not isfile(filename):
with open(filename, 'wb') as file:
# indicator
file.write(OP)
# root
file.write(pack('Q', 9))
self.root = IndexNode(is_leaf=True)
self.root.dump(file)
else:
with open(filename, 'rb+') as file:
if file.read(1) == OP:
file.close()
return BasicEngine.repair(filename)
else:
ptr = unpack('Q', file.read(8))[0]
file.seek(ptr)
self.root = IndexNode(file=file)
file.seek(0)
file.write(OP)
self.allocator = Allocator()
self.async_file = AsyncFile(filename)
self.command_que = SortedList()
self.file = open(filename, 'rb+', buffering=0)
self.lock = Lock()
self.on_interval = (0, 1)
self.on_write = False
self.task_que = TaskQue()
def malloc(self, size: int) -> int:
def is_inside(ptr: int) -> bool:
begin, end = self.on_interval
return begin <= ptr <= end or begin <= ptr + size <= end
ptr = self.allocator.malloc(size)
if ptr and is_inside(ptr):
self.free(ptr, size)
ptr = 0
if not ptr:
ptr = self.async_file.size
self.async_file.size += size
return ptr
def free(self, ptr: int, size: int):
self.allocator.free(ptr, size)
def time_travel(self, token: Task, node: IndexNode):
address = node.nth_value_ads(0)
for i in range(len(node.ptrs_value)):
ptr = self.task_que.get(token, address, node.ptr)
if ptr:
node.ptrs_value[i] = ptr
address += 8
if not node.is_leaf:
for i in range(len(node.ptrs_child)):
ptr = self.task_que.get(token, address, node.ptr)
if ptr:
node.ptrs_child[i] = ptr
address += 8
def a_command_done(self, token: Task):
token.command_num -= 1
if token.command_num == 0:
self.task_que.clean()
if not self.task_que.que and self.lock.locked():
self.lock.release()
# cum = cumulation
def do_cum(self, token: Task, free_nodes, command_map):
for node in free_nodes:
self.free(node.ptr, node.size)
for ptr, param in command_map.items():
data, depend = param if isinstance(param, tuple) else (param, 0)
self.ensure_write(token, ptr, data, depend)
self.time_travel(token, self.root)
self.root = self.root.clone()
def ensure_write(self, token: Task, ptr: int, data: bytes, depend=0):
async def coro():
while self.command_que:
ptr, token, data, depend = self.command_que.pop(0)
cancel = depend and self.task_que.is_canceled(token, depend)
if not cancel:
cancel = self.task_que.is_canceled(token, ptr)
if not cancel:
# 确保边界不相连
self.on_interval = (ptr - 1, ptr + len(data) + 1)
await self.async_file.write(ptr, data)
self.a_command_done(token)
self.on_write = False
if not self.on_write:
self.on_write = True
ensure_future(coro())
# 按ptr和token.id排序
self.command_que.append((ptr, token, data, depend))
token.command_num += 1
def close(self):
self.file.seek(0)
self.file.write(ED)
self.file.close()
self.async_file.close()
@staticmethod
def repair(filename: str):
temp = '__' + filename
size = getsize(filename)
with open(filename, 'rb') as file, open('$' + temp, 'wb') as items:
file.seek(9)
while file.tell() != size:
indicator = file.read(1)
if indicator != ED:
continue
with suppress(EOFError, UnpicklingError):
item = load(file)
if isinstance(item, tuple) and len(item) == 2:
dump(item, items)
rename('$' + temp, temp)
class ActivityDecorator(ActivityBase):
def __init__(self, target):
if not iscoroutinefunction(target):
raise TypeError("I find your lack of async disturbing.")
self.target = target #: target coroutine for this activity.
self.__trigger_obj = None
self.trigger_obj = trigger
self.mode = mode
self.lock = Lock()
# Create empty parameter dictionary
self.parameters = inspect.signature(target).parameters
self.empty_param_dict = {}
for param in self.parameters:
if param != "self":
self.empty_param_dict[param] = None
self._args = args
self._kwargs = kwargs
@property
def trigger_obj(self):
"""
Trigger connected to this activity.
:rtype: urban_journey.TriggerBase
"""
return self.__trigger_obj
@trigger_obj.setter
def trigger_obj(self, trigger):
if trigger is not None:
if self.__trigger_obj is not None:
self.__trigger_obj.remove_activity(self)
self.__trigger_obj = trigger
self.__trigger_obj.add_activity(self)
async def trigger(self, senders, sender_parameters, instance, *args, **kwargs):
"""
Called by the trigger.
:param senders: Dictionary with string typed key containing
"""
try:
if self.lock.locked():
if self.mode is ActivityMode.drop:
return
with (await self.lock):
# TODO: Remove support for "instance is None". This is currently only meant to be used in unittests.
# Create new parameters dictionary and fill it in with the data coming in from the triggers.
params = copy(self.empty_param_dict)
for param in params:
if param in sender_parameters:
params[param] = sender_parameters[param]
if instance is None:
await self.target(*args, *self._args, **kwargs, **self._kwargs, **params)
else:
await self.target(instance, *args, *self._args, **kwargs, **self._kwargs, **params)
except Exception as e:
# On exeption let the exception handler of the root node deal with it.
if instance is None:
print_exception(*sys.exc_info())
raise e
else:
instance.root.handle_exception(sys.exc_info())
def __call__(self, *args, **kwargs):
return self.target(*args, **kwargs)
class Actor(object):
"""
Main actor model.
Args:
inbox (GeneratorQueue): Inbox to consume from.
outbox (GeneratorQueue): Outbox to publish to.
loop (GeneratorQueue): Event loop.
"""
running = False
_force_stop = False
def __init__(self, inbox, outbox, loop=None):
self.inbox = inbox
self.outbox = outbox
if not loop:
loop = get_event_loop()
self._loop = loop
self._pause_lock = Lock(loop=self._loop)
self._stop_event = Event(loop=self._loop)
self._test = None
self.__testy = None
self.on_init()
@property
def paused(self):
"""Indicate if actor is paused."""
return self._pause_lock.locked()
async def start(self):
"""Main public entry point to start the actor."""
await self.initialize()
await self._start()
async def initialize(self):
"""Initialize the actor before starting."""
await self.on_start()
if self._force_stop:
return
if not self.running:
self.running = True
async def _start(self):
"""Run the event loop and force the on_stop event."""
try:
await self._run()
finally:
await self.on_stop()
async def resume(self):
"""Resume the actor."""
await self.on_resume()
self._pause_lock.release()
async def pause(self):
"""Pause the actor."""
await self._pause_lock.acquire()
await self.on_pause()
async def _block_if_paused(self):
"""Block on the pause lock."""
if self.paused:
await self._pause_lock.acquire()
await self._pause_lock.release()
async def _run(self):
"""Main event loop."""
while self.running:
await self._block_if_paused()
await self._process()
async def publish(self, data):
"""Push data to the outbox."""
await self.outbox.put(data)
async def stop(self):
"""Stop the actor."""
self.inbox = None
self.outbox = None
self.running = False
self._force_stop = True
self._stop_event.set()
try:
self._pause_lock.release()
except RuntimeError:
pass
async def _process(self):
"""Process incoming messages."""
if not self.inbox:
return
pending = {self.inbox.get(), self._stop_event.wait()}
result = await get_first_completed(pending, self._loop)
if self.running:
await self.on_message(result)
async def on_message(self, data):
"""Called when the actor receives a message."""
raise NotImplementedError
def on_init(self):
"""Called after the actor class is instantiated."""
pass
async def on_start(self):
"""Called before the actor starts ingesting the inbox."""
pass
async def on_stop(self):
"""Called after actor dies."""
pass
async def on_pause(self):
"""Called before the actor is paused."""
pass
async def on_resume(self):
"""Called before the actor is resumed."""
pass
def __init__(self):
self.items = []
self.lock = Lock()
class FakeAioRedisConnection:
def __init__(self, max_age_seconds=120, loop=None):
assert max_age_seconds >= 1
self.store = {}
self.max_age = max_age_seconds
self.lock = Lock()
self.closed = False
self.loop = loop or get_event_loop()
self.clean_keys()
async def pexpire(self, key, pexpire):
self.expire(key, pexpire / 1000)
async def expire(self, key, expire):
with (await self.lock):
item = self.store.get(key, None)
if not item:
return None
self.store[key] = (item[0], time.time() + expire)
async def persist(self, key):
with (await self.lock):
item = self.store.get(key, None)
if not item:
return None
self.store[key] = (item[0], float('inf'))
async def set(self, key, value, pexpire=None):
with (await self.lock):
if not pexpire:
self.store[key] = (
str(value).encode(), float('inf'))
else:
self.store[key] = (
str(value).encode(), time.time() + pexpire / 1000)
async def get(self, key):
with (await self.lock):
item = self.store.get(key, None)
if not item:
return None
return item[0]
async def delete(self, key, default=None):
with (await self.lock):
item = self.store.get(key, None)
if not item:
return None
del self.store[key]
return item[0]
async def lpush(self, key, value):
with (await self.lock):
li, timeout = self.store.get(key, (None, None))
if not li:
self.store[key] = (deque([value]), float('inf'))
else:
li.appendleft(value)
self.store[key] = (li, timeout)
return
async def lpushx(self, key, value):
with (await self.lock):
li, timeout = self.store.get(key, (None, None))
if not li:
return None
li.appendleft(value)
self.store[key] = (li, timeout)
return
async def rpush(self, key, value):
with (await self.lock):
li, timeout = self.store.get(key, (None, None))
if not li:
self.store[key] = (deque([value]), float('inf'))
else:
li.append(value)
self.store[key] = (li, timeout)
return
async def rpushx(self, key, value):
with (await self.lock):
li, timeout = self.store.get(key, (None, None))
if not li:
return None
li.append(value)
self.store[key] = (li, timeout)
return
async def lpop(self, key):
with (await self.lock):
li, _ = self.store.get(key, (None, None))
if not li:
return None
elem = li.popleft()
return elem
async def rpop(self, key):
with (await self.lock):
li, _ = self.store.get(key, (None, None))
if not li:
return None
elem = li.pop()
return elem
async def lrange(self, key, start, end):
with (await self.lock):
li, _ = self.store.get(key, (None, None))
if not li:
return []
if start == 0 and end == -1:
return li
return li[start, end]
async def llen(self, key):
with (await self.lock):
li, _ = self.store.get(key, (None, None))
if not li:
return None
return len(li)
async def incr(self, key):
with (await self.lock):
value, timeout = self.store.get(key, (0, float('inf')))
value += 1
self.store[key] = (value, timeout)
async def incrby(self, key, increment):
with (await self.lock):
value, timeout = self.store.get(key, (0, float('inf')))
value += increment
self.store[key] = (value, timeout)
async def quit(self):
with (await self.lock):
self.closed = True
self.store = {}
return
def clean_keys(self):
if self.closed:
return
for k in list(self.store.keys()):
if not self.lock.locked():
item = self.store[k]
if time.time() > item[1]:
del self.store[k]
self.loop.call_soon_threadsafe(
self.loop.call_later,
self.max_age,
self.clean_keys)