def _download(self, canal, seccion, season, titulo, url, cb_progress):
"""Download an episode to disk."""
self.cancelled = False
# levantamos el browser
qinput = DeferredQueue()
bquit = Event()
self.browser_quit.add(bquit)
authuser = config.get('user', '')
authpass = config.get('password', '')
# build where to save it
fname, tempf = self._setup_target(canal, seccion, season,
titulo, u".avi")
logger.debug("Downloading to temporal file %r", tempf)
logger.info("Download episode %r: browser started", url)
brow = MiBrowser(self, authuser, authpass, url, tempf, qinput, bquit)
brow.start()
# loop reading until finished
self._prev_progress = None
logger.info("Downloader started receiving bytes")
while True:
# get all data and just use the last item
payload = yield qinput.deferred_get()
if self.cancelled:
logger.debug("Cancelled! Quit browser, wait, and clean.")
bquit.set()
yield qinput.deferred_get()
if os.path.exists(tempf):
os.remove(tempf)
logger.debug("Cancelled! Cleaned up.")
raise CancelledError()
# special situations
if payload is None:
# no data, let's try again
continue
data = payload[-1]
if isinstance(data, Exception):
raise data
if data == DONE_TOKEN:
break
# actualizamos si hay algo nuevo
if data != self._prev_progress:
cb_progress(data)
self._prev_progress = data
# movemos al nombre correcto y terminamos
logger.info("Downloading done, renaming temp to %r", fname)
os.rename(tempf, fname)
self.browser_quit.remove(bquit)
defer.return_value(fname)
def get_uid_from_dbus_name(dbus_name, bus=None):
"""Return a deferred that gets the uid of the user owning the given
system D-Bus name.
"""
if not bus:
bus = dbus.SystemBus()
pid = yield get_pid_from_dbus_name(dbus_name, bus)
with open("/proc/%s/status" % pid) as proc:
values = [v for v in proc.readlines() if v.startswith("Uid:")]
uid = int(values[0].split()[1])
return_value(uid)
def get_uid_from_dbus_name(dbus_name, bus=None):
"""Return a deferred that gets the uid of the user owning the given
system D-Bus name.
"""
if not bus:
bus = dbus.SystemBus()
pid = yield get_pid_from_dbus_name(dbus_name, bus)
with open("/proc/%s/status" % pid) as proc:
values = [v for v in proc.readlines() if v.startswith("Uid:")]
uid = int(values[0].split()[1])
return_value(uid)
def from_update(cls, update):
"""Securely parse from update, or return a not useful item."""
try:
item = yield cls._from_update(update)
except Exception as exc:
logger.exception("Problem parsing message from update: %s", exc)
item = None
if item is None:
# bad parsing, crash in _from_update, user not allowed, etc
update_id = int(update['update_id'])
item = cls(message_id=update_id, useful=False)
defer.return_value(item)
def get_proc_info_from_dbus_name(dbus_name, bus=None):
"""Return a deferred that gets the pid, the uid of the user owning the
given system D-Bus name and its command line.
"""
if not bus:
bus = dbus.SystemBus()
pid = yield get_pid_from_dbus_name(dbus_name, bus)
with open("/proc/%s/status" % pid) as proc:
values = [v for v in proc.readlines() if v.startswith("Uid:")]
with open("/proc/%s/cmdline" % pid) as cmdline_file:
cmdline = cmdline_file.read()
uid = int(values[0].split()[1])
return_value((pid, uid, cmdline))
def _get(self, filename):
"""Get the content from the server or a local source."""
if self.update_source is None:
# from the server
url = BACKENDS_BASE_URL + filename
logger.debug("Getting content from url %r", url)
_, content = yield utils.download(url)
else:
# from a local source
filepath = os.path.join(self.update_source, filename)
logger.debug("Getting content from filepath %r", filepath)
with open(filepath, 'rb') as fh:
content = fh.read()
defer.return_value(content)
def get_proc_info_from_dbus_name(dbus_name, bus=None):
"""Return a deferred that gets the pid, the uid of the user owning the
given system D-Bus name and its command line.
"""
if not bus:
bus = dbus.SystemBus()
pid = yield get_pid_from_dbus_name(dbus_name, bus)
with open("/proc/%s/status" % pid) as proc:
values = [v for v in proc.readlines() if v.startswith("Uid:")]
# instead of ", encoding='utf8'" we use the "rb"/decode() here for
# py2 compatibility
with open("/proc/%s/cmdline" % pid, "rb") as cmdline_file:
cmdline = cmdline_file.read().decode("utf-8")
uid = int(values[0].split()[1])
return_value((pid, uid, cmdline))
def _episode_download(self, episode):
"""Effectively download an episode."""
logger.debug("Effectively downloading episode %s", episode.episode_id)
self.episodes_download.start()
# download!
downloader = self.downloaders[episode.downtype]
season = getattr(episode, 'season', None) # wasn't always there
fname = yield downloader.download(
episode.channel, episode.section, season, episode.title,
episode.url, self.episodes_download.progress)
episode_name = u"%s - %s - %s" % (episode.channel, episode.section,
episode.composed_title)
notify(u"Descarga finalizada", episode_name)
defer.return_value((fname, episode))
def _episode_download(self, episode):
"""Effectively download an episode."""
logger.debug("Effectively downloading episode %s", episode.episode_id)
self.episodes_download.start()
# download!
downloader = self.downloaders[episode.downtype]
season = getattr(episode, 'season', None) # wasn't always there
fname = yield downloader.download(episode.channel, episode.section,
season, episode.title, episode.url,
self.episodes_download.progress)
episode_name = u"%s - %s - %s" % (episode.channel, episode.section,
episode.composed_title)
notify(u"Descarga finalizada", episode_name)
defer.return_value((fname, episode))
def _from_update(cls, update):
"""Create from a telegram message."""
try:
msg = update['message']
except KeyError:
logger.warning("Unknown update type: %r", update)
return
# only allow messages from a specific user (set in the first message, can be reset
# through config)
from_id = msg['from']['id']
allowed_user = config.USER_ALLOWED
if allowed_user is None:
# user not configured yet, let's set it now
logger.info("Setting allowed user id to %r", from_id)
config.USER_ALLOWED = from_id
config.save()
else:
# check user is allowed
if allowed_user != from_id:
logger.warning("Ignoring message from user not allowed: %r", from_id)
return
if 'text' in msg:
text = msg['text']
info = dict(text=text)
elif 'photo' in msg:
# grab the content of the biggest photo only
photo = max(msg['photo'], key=lambda photo: photo['width'])
extfile_path = yield download_file(photo['file_id'])
media_type = cls.MEDIA_TYPE_IMAGE
text = msg['caption'] if 'caption' in msg else None
info = dict(extfile_path=extfile_path, media_type=media_type, text=text)
elif 'voice' in msg:
# get the audio file
extfile_path = yield download_file(msg['voice']['file_id'])
media_type = cls.MEDIA_TYPE_AUDIO
info = dict(extfile_path=extfile_path, media_type=media_type)
else:
logger.warning("Message type not supported: %s", msg)
return
update_id = int(update['update_id'])
sent_at = datetime.fromtimestamp(msg['date'])
defer.return_value(cls(sent_at=sent_at, message_id=update_id, **info))
def poem_main():
addresses=parse_args()
xform_addr=addresses.pop(0)
proxy=TransformProxy(*xform_addr)
results=[]
@defer.inline_callbacks
def get_transformed_poem(host,port):
try:
poem=yield get_poetry(host,port)
except Exception,e:
print >>sys.stderr,'The poem download failed',e
raise
try:
poem=yield proxy.xform('cummingsify',poem)
except Exception:
print >>sys.stderr,'Cummingsify failed'
raise
defer.return_value(poem)
def download_file(file_id):
"""Download the file content from Telegram."""
url = build_baseapi_url('getFile', file_id=file_id)
logger.debug("Getting file path, file_id=%s", file_id)
downloader = _Downloader(url)
encoded_data = yield downloader.deferred
logger.debug("getFile response encoded data len=%d", len(encoded_data))
data = json.loads(encoded_data.decode('utf8'))
if not data.get('ok'):
logger.warning("getFile result is not ok: %s", encoded_data)
return
remote_path = data['result']['file_path']
url = build_fileapi_url(remote_path)
file_path = os.path.join(data_basedir, uuid.uuid4().hex + '-' + os.path.basename(remote_path))
logger.debug("Getting file content, storing in %r", file_path)
downloader = _Downloader(url, file_path)
downloaded_size = yield downloader.deferred
logger.debug("Downloaded file content, size=%d", downloaded_size)
defer.return_value(file_path)
def _invoke(self, *args, **kwargs):
"""
:rtype: L{defer.Deferred} of C{unicode}
"""
request = self._create_request(*args, **kwargs)
self._auto_finish = False
try:
content = yield maybe_deferred(self.func, request)
except Exception as e:
exception_cls, exception, traceback = sys.exc_info()
try:
resolver = self.spec.resolve_exception(e)
except LookupError:
IOLoop.instance().add_callback(self.send_error, 500)
raise exception_cls, exception, traceback
else:
if not isinstance(resolver, callable):
status_code, message = resolver
IOLoop.instance().add_callback(self.finish)
raise HTTPError(status_code, e, reason=message)
content = resolver(e, exception_cls, exception, traceback)
content, headers = yield maybe_deferred(self.spec.render, request, content)
return_value((content, headers))
def _invoke(self, *args, **kwargs):
"""
:rtype: L{defer.Deferred} of C{unicode}
"""
request = self._create_request(*args, **kwargs)
self._auto_finish = False
try:
content = yield maybe_deferred(self.func, request)
except Exception as e:
exception_cls, exception, traceback = sys.exc_info()
try:
resolver = self.spec.resolve_exception(e)
except LookupError:
IOLoop.instance().add_callback(self.send_error, 500)
raise exception_cls, exception, traceback
else:
if not isinstance(resolver, callable):
status_code, message = resolver
IOLoop.instance().add_callback(self.finish)
raise HTTPError(status_code, e, reason=message)
content = resolver(e, exception_cls, exception, traceback)
content, headers = yield maybe_deferred(self.spec.render, request,
content)
return_value((content, headers))
def get_network_state(self):
"""Query NetworkManager about the network state."""
state = pk_enums.NETWORK_UNKNOWN
try:
active_conns = yield self.get_dbus_property(self.proxy,
self.NM_DBUS_IFACE,
"ActiveConnections")
except dbus.DBusException:
log.warning("Failed to determinate network state")
return_value(state)
for conn in active_conns:
conn_obj = self.bus.get_object(self.NM_DBUS_IFACE, conn)
try:
is_default = yield self.get_dbus_property(conn_obj,
self.NM_ACTIVE_CONN_DBUS_IFACE,
"Default")
if not is_default:
continue
devs = yield self.get_dbus_property(conn_obj,
self.NM_ACTIVE_CONN_DBUS_IFACE,
"Devices")
except dbus.DBusException:
log.warning("Failed to determinate network state")
break
priority_device_type = -1
for dev in devs:
try:
dev_obj = self.bus.get_object(self.NM_DBUS_IFACE, dev)
dev_type = yield self.get_dbus_property(dev_obj,
self.NM_DEVICE_DBUS_IFACE,
"DeviceType")
except dbus.DBusException:
log.warning("Failed to determinate network state")
return_value(pk_enums.NETWORK_UNKNOWN)
# prioterizse device types, since a bridged GSM/CDMA connection
# should be returned as a GSM/CDMA one
# The NM_DEVICE_TYPE_* enums are luckly ordered in this sense.
if dev_type <= priority_device_type:
continue
priority_device_type = dev_type
if dev_type in (self.NM_DEVICE_TYPE_GSM,
self.NM_DEVICE_TYPE_CDMA):
state = pk_enums.NETWORK_MOBILE
elif dev_type == self.NM_DEVICE_TYPE_ETHERNET:
state = pk_enums.NETWORK_WIRED
elif dev_type == self.NM_DEVICE_TYPE_WIFI:
state = pk_enums.NETWORK_WIFI
elif dev_type == self.NM_DEVICE_TYPE_UNKNOWN:
state = pk_enums.NETWORK_OFFLINE
else:
state = pk_enums.NETWORK_ONLINE
return_value(state)
def _query_network_manager(self):
"""Query NetworkManager about the network state."""
state = pk.NetworkEnum.OFFLINE
try:
active_conns = yield self.get_dbus_property(self.proxy,
self.NM_DBUS_IFACE,
"ActiveConnections")
except dbus.DBusException:
log.warning("Failed to determinate network state")
return_value(state)
for conn in active_conns:
conn_obj = self.bus.get_object(self.NM_DBUS_IFACE, conn)
try:
is_default = yield self.get_dbus_property(
conn_obj, self.NM_ACTIVE_CONN_DBUS_IFACE, "Default")
if not is_default:
continue
devs = yield self.get_dbus_property(
conn_obj, self.NM_ACTIVE_CONN_DBUS_IFACE, "Devices")
except dbus.DBusException:
log.warning("Failed to determinate network state")
break
priority_device_type = -1
for dev in devs:
try:
dev_obj = self.bus.get_object(self.NM_DBUS_IFACE, dev)
dev_type = yield self.get_dbus_property(
dev_obj, self.NM_DEVICE_DBUS_IFACE, "DeviceType")
except dbus.DBusException:
log.warning("Failed to determinate network state")
return_value(pk.NetworkEnum.UNKNOWN)
# prioterizse device types, since a bridged GSM/CDMA connection
# should be returned as a GSM/CDMA one
# The NM_DEVICE_TYPE_* enums are luckly ordered in this sense.
if dev_type <= priority_device_type:
continue
priority_device_type = dev_type
if dev_type in (self.NM_DEVICE_TYPE_GSM,
self.NM_DEVICE_TYPE_CDMA):
state = pk.NetworkEnum.MOBILE
elif dev_type == self.NM_DEVICE_TYPE_ETHERNET:
state = pk.NetworkEnum.WIRED
elif dev_type == self.NM_DEVICE_TYPE_WIFI:
state = pk.NetworkEnum.WIFI
elif dev_type == self.NM_DEVICE_TYPE_UNKNOWN:
state = pk.NetworkEnum.OFFLINE
else:
state = pk.NetworkEnum.ONLINE
return_value(state)
def _download(self, canal, seccion, season, titulo, url, cb_progress):
"""Download an episode to disk."""
url = str(url)
logger.info("Download episode %r", url)
# build where to save it
fname, tempf = self._setup_target(canal, seccion,
season, titulo, self.file_extension)
logger.debug("Downloading to temporal file %r", tempf)
fh = open(tempf, "wb")
def report(dloaded, total):
"""Report download."""
if total == -1:
m = "%d MB" % (dloaded // MB,)
else:
size_mb = total // MB
perc = dloaded * 100.0 / total
m = "%.1f%% (de %d MB)" % (perc, size_mb)
if m != self._prev_progress:
cb_progress(m)
self._prev_progress = m
def save():
"""Save available bytes to disk."""
data = req.read(req.bytesAvailable())
fh.write(data)
request = QtNetwork.QNetworkRequest()
request.setUrl(QtCore.QUrl(url))
for hk, hv in self.headers.items():
request.setRawHeader(hk, hv)
def end_ok():
"""Finish Ok politely the deferred."""
if not self.downloader_deferred.called:
self.downloader_deferred.callback(True)
def end_fail(exc):
"""Finish in error politely the deferred."""
if not self.downloader_deferred.called:
self.downloader_deferred.errback(exc)
deferred = self.downloader_deferred = defer.Deferred()
req = self.manager.get(request)
req.downloadProgress.connect(report)
req.error.connect(end_fail)
req.readyRead.connect(save)
req.finished.connect(end_ok)
try:
yield deferred
except Exception as err:
logger.debug("Exception when waiting deferred: %s (request "
"finished? %s)", err, req.isFinished())
if not req.isFinished():
logger.debug("Aborting QNetworkReply")
req.abort()
raise
finally:
fh.close()
# rename to final name and end
logger.info("Downloading done, renaming temp to %r", fname)
os.rename(tempf, fname)
defer.return_value(fname)
def http_jsonify_dict_decorator(*args, **kwargs):
obj_dict = yield defer(func, *args, **kwargs)
return_value(encode_dict(obj_dict))
def http_jsonify_decorator(*args, **kwargs):
obj = yield defer(func, *args, **kwargs)
return_value(encode_func(obj))
def get_network_state(self):
"""Update the network state."""
return_value(self._state)
def get_network_state(self):
"""Update the network state."""
self.state = yield self._query_network_manager()
return_value(self.state)
def get_network_state(self):
"""Update the network state."""
self.state = self._parse_route_file()
return_value(self.state)
def http_jsonify_decorator(*args, **kwargs):
obj = yield defer(func, *args, **kwargs)
return_value(encode_func(obj))
def http_jsonify_dict_decorator(*args, **kwargs):
obj_dict = yield defer(func, *args, **kwargs)
return_value(encode_dict(obj_dict))
