def test_set_missing_key(self):
"Expected to throw"
cache = CacheManager()
with raises(ReferenceError):
cache.set(key=None, value='value', expiration=1)
def main():
yt_resource = build('youtube', 'v3', developerKey=config['api_key'])
print_info(
f"If you want to ignore some files, add name of those files in {config['exceptions_file']}"
f" in the same folder in which files to be renamed are present.")
files = ls()
exceptions = get_exceptions()
cache = CacheManager(config)
if cache.is_local_playlist_cache_available():
playlist_id = get_playlist_id_from_cache(cache)
if playlist_id is None:
playlist_id = get_playlist_id_from_youtube(cache, yt_resource)
else:
playlist_id = get_playlist_id_from_youtube(cache, yt_resource)
playlist = YTPlaylist(yt_resource, playlist_id)
playlist.fetch_videos()
playlist.print_videos()
remote_serial_dict = playlist.get_videos_serial()
print()
renaming_helper = RenamingHelper(
remote_serial_dict,
files,
exceptions,
character_after_serial=config['character_after_serial'])
rename(renaming_helper, cache)
def test_get_missing_key(self):
"Expected to throw"
cache = CacheManager()
with raises(ReferenceError):
cache.get(key=None)
def test_set_missing_value(self):
"Expected to set an empty key"
cache = CacheManager()
cache.set(key='key', value=None, expiration=1)
assert not cache.get('key')
def test_set_has_value(self):
"Expected to find the required value"
cache = CacheManager()
cache.set(key='key', value='value', expiration=1)
assert cache.get('key') == 'value'
def test_get_existent_key(self):
"Expected to return None"
cache = CacheManager()
key = 'key'
value = 'value'
cache.set(key, value)
assert cache.get(key) == value
def get_playlist_id_from_youtube(cache: CacheManager, yt_resource: Resource):
channel_id = get_channel_id(cache, yt_resource)
channel = YTChannel(yt_resource, channel_id)
channel.fetch_playlists()
channel.print_playlists()
playlist_cache_unit = channel.select_playlist(
int(input_in_range('Select playlist: ', 1,
channel.total_playlists + 1)))
cache.update_playlist_cache(playlist_cache_unit)
return playlist_cache_unit['id']
def get_playlist_id_from_cache(cache: CacheManager):
playlist_cache_unit = cache.local_playlist_cache.list()[0]
print_info(
f"Playlist: \"{playlist_cache_unit['title']}\" found in {os.path.join(os.path.basename(config['local_cache']),'playlist.json')}."
)
inp = non_empty_input('Is the playlist correct?(n if no): ').lower()
if inp in ('n', 'no'):
cache.delete_local_playlist_cache()
print()
return None
return playlist_cache_unit['id']
def rename(renaming_helper: RenamingHelper, cache: CacheManager):
renaming_helper.generate_rename_dict()
if renaming_helper.is_rename_dict_formed():
cache.save()
renaming_helper.dry_run()
print('\n')
confirm = non_empty_input(
"This can't be undone. Are you sure to rename?(y/n): ").lower()
print()
if confirm == "y" or confirm == "yes":
renaming_helper.start_batch_rename()
else:
print("Nothing renamed")
else:
print_info(
'\nFiles already have serial number or match not found in local files and youtube videos.'
)
print_info('Check if you have selected correct playlist.')
def __init__(self):
"""
Init method.
"""
# class variables
self._as = None # SimpleActionServer variable
self._goal = "" # Goal a recibir
self._goal_exec = False # Indicates if goal is being handled
self._pause = False # Indicates if pause
self._step = 'init' # Init, rss_reader, show_info
self._out = False # Varable to exit the while
# Goal varaibles
self._max_time = 0
self._number_plays = 0
self._t0 = 0
self._t1 = 0
self._time_run = 0
self._i_plays = 0
# Local paths
rospack = rospkg.RosPack()
self._root_path = rospack.get_path(pkg_name) # Package path
self._data_path = self._root_path + '/data/' # Data path
self._news_cache = self._data_path + 'news_cache.txt' # Cache file
# Tablet paths
self._default_path = '/default.png' # Default image
# Rss objects
self._feed_name = 'tu_tiempo'
self._category_name = 'cover_page'
#self._rss_feed = self.feeds[self._feed_name][self._category_name] # Feed url
# Cache object
self._cache_manager = CacheManager(self._data_path)
# init the skill
Skill.__init__(self, skill_name, CONDITIONAL)
class CacheTest(unittest.TestCase):
def setUp(self):
self.cache_manager = CacheManager()
def test_use_cache(self):
key1 = "honda"
self.cache_manager.add(key1, Car(key1, 32000))
car = self.cache_manager.get(key1)
self.assertEqual(key1, car.brand)
self.assertEqual(32000, car.price)
self.cache_manager.remove(key1)
self.assertIsNone(self.cache_manager.get(key1))
def test_can_expire_cache(self):
key1 = "mykey"
self.cache_manager.add(key1, 'Value', expiry_time=timedelta(minutes=2))
value = self.cache_manager.get(key1,
current_time=datetime.now() +
timedelta(minutes=5))
self.assertIsNone(value)
async def main():
cache_manager = CacheManager()
truly_awesome_bank_API_client = TrulyAwesomeBankAPIClient()
resource_manager = ResourceManager(cache_manager,
truly_awesome_bank_API_client)
transaction = {'type': 'PAYMENT', 'amount': 100, 'currency': 'EUR'}
print('=======================')
print('| WRITE THROUGH |')
print('=======================')
print('>>> Save transaction')
entry = await resource_manager.save_with_write_through(transaction)
print('>>> Get transaction')
await resource_manager.fetch_transaction_by_id(entry['id'])
print('=======================')
print('| WRITE BEHIND |')
print('=======================')
print('>>> Save transaction')
entry = await resource_manager.save_with_write_behind(transaction)
print('>>> Get transaction')
await resource_manager.fetch_transaction_by_id(entry['id'])
print('')
print('--------------------------------------------')
print('| AWESOME BANK DATABASE (before sync) |')
print('--------------------------------------------')
print(truly_awesome_bank_API_client._TrulyAwesomeBankAPIClient__database)
print('')
# wait for synchronization
await asyncio.sleep(10)
print('')
print('--------------------------------------------')
print('| AWESOME BANK DATABASE (after sync) |')
print('--------------------------------------------')
print(truly_awesome_bank_API_client._TrulyAwesomeBankAPIClient__database)
print('')
def get_channel_id(cache: CacheManager, yt_resource: Resource):
def get_from_yt(user_input):
youtube = Youtube(yt_resource)
youtube.search_channel(user_input)
youtube.print_channels()
return youtube.select_channel(
int(
input_in_range('Select channel: ', 1,
youtube.total_channels + 1)))
if cache.is_local_channel_cache_available():
channel_cache_unit = cache.local_channel_cache.list()[0]
print_info(
f"Channel {channel_cache_unit['title']} found in {os.path.join(os.path.basename(config['local_cache']),'channel.json')}."
)
inp = non_empty_input('Is the channel correct?(n if no): ').lower()
if inp in ('n', 'no'):
cache.delete_local_channel_cache()
print()
return get_channel_id(cache, yt_resource)
elif cache.is_shared_channel_cache_available():
cache.shared_channel_cache.print()
user_input = non_empty_input(
'Select or enter channel name to search on youtube: ')
if user_input.isdigit():
channel_cache_unit = cache.shared_channel_cache.list()[
int(user_input) - 1]
else:
channel_cache_unit = get_from_yt(user_input)
else:
user_input = non_empty_input('Search channel on youtube: ')
channel_cache_unit = get_from_yt(user_input)
cache.update_channel_cache(channel_cache_unit)
cache.save_shared_channel_cache()
return channel_cache_unit['id']
def main():
setting = SettingManager()
args = parser.parse_args()
if args.dry:
environ['DRY'] = 'True'
if args.once:
__once(setting.users[0], setting.common, args.once)
return
cache = CacheManager()
user_count = len(setting.users)
if user_count > 1:
with ThreadPoolExecutor(user_count) as executor:
futures = []
try:
for user in setting.users:
futures.append(
executor.submit(__parallel_process, user,
setting.common, args.start, cache))
executor.shutdown()
except KeyboardInterrupt:
LOGGER.warning('shutdown')
else:
__parallel_process(setting.users[0], setting.common, args.start, cache)
import random
import string
from random import randint
from event_producer import EventProducer
from cache_manager import CacheManager
if __name__ == "__main__":
publisher = EventProducer()
publisher.connect()
#Pushing random items
item_name = "item" + str(randint(0, 9999999999))
publisher.publish_event("list/mylist", item_name)
print("Data in cache")
cache = CacheManager()
cache.print_data("mylist")
sys.path.append(os.path.abspath(os.path.join('..', '')))
from db_manager import master_list
from cache_manager import CacheManager
from config import device_id
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
logging.basicConfig(level=logging.INFO)
handler = logging.FileHandler('webapp_log.log')
handler.setFormatter(formatter)
logger = logging.getLogger(__name__)
logger.addHandler(handler)
app = Flask(__name__, static_folder="./static", template_folder="./static")
ch = CacheManager()
@app.route("/list")
def get_list():
return "Master List:{}".format(master_list)
#return render_template("index.html")
@app.route("/ping")
def hello():
return "PONG"
@app.route("/mylist", methods=["GET"])
def get_my_list():
def setUp(self):
self.cache_manager = CacheManager()
# settings
input_file_name = 'read.html'
cache_dir_name = 'cache'
out_file_name = 'out.csv'
min_delay = 90
max_delay = 120
print('Load books from file: "%s"' % input_file_name)
read_parser = ReadParser()
if read_parser.load_from_file(input_file_name) is False:
exit(1)
print('Books loaded.')
print('Parse books from summary.')
books = read_parser.parse_books()
print('Books parsed: %s.' % len(books))
print('Start download detailed book pages.')
cache = CacheManager(cache_dir_name)
loader = PageLoader(cache, min_delay, max_delay)
loader.download(books)
print('Detailed book pages downloaded.')
print('Prepare books for export.')
details_parser = DetailsParser(cache)
ready_books = details_parser.parse(books)
print('Books ready to export: %s.' % len(ready_books))
writer = CsvWriter()
writer.save(ready_books, out_file_name)
print('Books saved to "%s"' % out_file_name)
class CacheFs(fuse.Fuse):
def __init__(self, *args, **kw):
fuse.Fuse.__init__(self, *args, **kw)
self.cfg = None
self.cacheManager = None
self.filebuffer = None
def run(self):
self.cacheManager = CacheManager(self.cfg.cache_manager)
self.main()
def stop(self):
pass
def parse(self, *args, **kw):
'''This method shall be moved somewhere in config module'''
fuse_args = super(CacheFs, self).parse(*args, **kw)
if fuse_args.modifiers['showhelp']:
# not beautiful, but works
sys.exit(0)
self.cfg = config_canonical.getConfig()
options, arguments = self.cmdline
loclogger.initialize(options.log_path)
if options.debug:
loclogger.enableDebug()
self.cfg.parse(options, arguments, self.fuse_args.mountpoint)
@trace
def fsinit(self):
INFO("Initializing file system")
@trace
def fsdestroy(self):
self.stop()
INFO("Unmounting file system")
@trace
def statfs(self):
stats = fuse.StatVfs()
return stats
@trace
def getattr(self, path):
st = self.cacheManager.getAttributes(path)
if not st:
return -errno.ENOENT
return st
@trace
def access(self, path, flags):
if flags == os.F_OK:
if self.cacheManager.exists(path):
return 0
else:
return -errno.EACCES
if flags & os.W_OK:
return -errno.EACCES
return 0
@trace
def readlink(self, path):
pathToCachedFile = self.cacheManager.readLink(path)
if pathToCachedFile:
return pathToCachedFile
return -errno.ENOENT
@trace
def opendir(self, path):
if not (self.cacheManager.exists(path) and self.cacheManager.isDirectory(path)):
return -errno.ENOENT
return None # success
@trace
def readdir(self, path, offset = None, dh = None):
# TODO: Update timestamps: readdir updates atime
if not self.cacheManager.exists(path):
yield
elif not self.cacheManager.isDirectory(path):
yield
yield fuse.Direntry(".")
yield fuse.Direntry("..")
for entry in self.cacheManager.listDirectory(path):
yield fuse.Direntry(entry)
@trace
def read(self, path, size, offset, fh):
if fh.lseek != offset:
os.lseek(fh.fh, offset, 0)
self.filebuffer = os.read(fh.fh, size)
fh.lseek = offset + size
return self.filebuffer
@trace
def open(self, path, flags):
st = self.cacheManager.getAttributes(path)
if st:
cache_path = self.cacheManager.getPathToCachedFile(path)
return File(os.open(cache_path, flags), os.path.basename(path), st)
@trace
def release(self, path, flags, fh):
os.close(fh.fh)
@trace
def fgetattr(self, path, fh):
return fh.stat
@trace
def fsync(self, path, datasync, fh):
pass
@trace
def flush(self, path, fh):
pass
@trace
def mknod(self, path, mode, rdev):
return -errno.ENOENT
@trace
def mkdir(self, path, mode):
return -errno.ENOENT
@trace
def unlink(self, path):
return -errno.ENOENT
@trace
def rmdir(self, path):
return -errno.ENOENT
@trace
def symlink(self, target, name):
return -errno.EOPNOTSUPP
@trace
def link(self, target, name):
return -errno.EOPNOTSUPP
@trace
def rename(self, old, new):
return -errno.EOPNOTSUPP
@trace
def chmod(self, path, mode):
return -errno.EOPNOTSUPP
@trace
def chown(self, path, uid, gid):
return -errno.EOPNOTSUPP
@trace
def truncate(self, path, size):
return 0
@trace
def releasedir(self, path, dh = None):
pass
@trace
def fsyncdir(self, path, datasync, dh):
pass
@trace
def write(self, path, buf, offset, fh):
return -errno.EOPNOTSUPP
@trace
def ftruncate(self, path, size, fh):
fh.truncate(size)
def run(self):
self.cacheManager = CacheManager(self.cfg.cache_manager)
self.main()
class NewsSkill(Skill):
"""
News skill class.
"""
# Feedback and result of this skill
_feedback = news_skill.msg.NewsFeedback()
_result = news_skill.msg.NewsResult()
# Node constants
# Feeds
feeds = {
'bbc': {
'cover_page': 'http://www.bbc.co.uk/mundo/index.xml',
'last_news':
'http://www.bbc.co.uk/mundo/ultimas_noticias/index.xml',
'spain': '',
'international':
'http://www.bbc.co.uk/mundo/temas/internacional/index.xml',
'opinion': '',
'sports': '',
'technology':
'http://www.bbc.co.uk/mundo/temas/tecnologia/index.xml',
'science': 'http://www.bbc.co.uk/mundo/temas/ciencia/index.xml',
'culture': 'http://www.bbc.co.uk/mundo/temas/cultura/index.xml'
},
'europa_press': {
'cover_page': 'https://www.europapress.es/rss/rss.aspx',
'last_news': '',
'spain': 'https://www.europapress.es/rss/rss.aspx?ch=00066',
'international':
'https://www.europapress.es/rss/rss.aspx?ch=00069',
'opinion': '',
'sports': 'https://www.europapress.es/rss/rss.aspx?ch=00067',
'technology': 'https://www.europapress.es/rss/rss.aspx?ch=00564',
'science': '',
'culture': 'https://www.europapress.es/rss/rss.aspx?ch=00126'
},
'libertad_digital': {
'cover_page':
'http://feeds2.feedburner.com/libertaddigital/portada',
'last_news': '',
'spain': 'http://feeds2.feedburner.com/libertaddigital/nacional',
'international':
'http://feeds2.feedburner.com/libertaddigital/internacional',
'opinion': 'http://feeds2.feedburner.com/libertaddigital/opinion',
'sports': 'http://feeds2.feedburner.com/libertaddigital/deportes',
'technology':
'http://feeds2.feedburner.com/libertaddigital/internet',
'science': '',
'culture': 'http://feeds.feedburner.com/libertaddigital/cultura'
},
'el_pais': {
'cover_page': 'http://ep00.epimg.net/rss/elpais/portada.xml',
'last_news': 'http://ep00.epimg.net/rss/tags/ultimas_noticias.xml',
'spain': 'http://ep00.epimg.net/rss/politica/portada.xml',
'international':
'http://ep00.epimg.net/rss/internacional/portada.xml',
'opinion': 'http://ep00.epimg.net/rss/elpais/opinion.xml',
'sports': 'http://ep00.epimg.net/rss/deportes/portada.xml',
'technology': 'http://ep00.epimg.net/rss/tecnologia/portada.xml',
'science': 'http://ep00.epimg.net/rss/elpais/ciencia.xml',
'culture': 'http://ep00.epimg.net/rss/cultura/portada.xml'
},
'tu_tiempo': {
'cover_page': 'http://xml.tutiempo.net/xml/3768.xml'
}
}
feeds_no = {
# No aptos
'20_minutos': { # Resumenes largos
'cover_page': 'https://www.20minutos.es/rss/',
'last_news': '',
'spain': 'https://www.20minutos.es/rss/nacional/',
'international': 'https://www.20minutos.es/rss/internacional/',
'opinion': 'https://www.20minutos.es/rss/opiniones/',
'sports': 'https://www.20minutos.es/rss/deportes/',
'technology': 'https://www.20minutos.es/rss/tecnologia/',
'science': 'https://www.20minutos.es/rss/ciencia/',
'culture': 'https://www.20minutos.es/rss/cultura/'
},
'el_mundo': { # Sin resumenes
'cover_page':
'https://e00-elmundo.uecdn.es/elmundo/rss/portada.xml',
'last_news': '',
'spain': 'https://e00-elmundo.uecdn.es/elmundo/rss/espana.xml',
'international':
'https://e00-elmundo.uecdn.es/elmundo/rss/internacional.xml',
'opinion': '',
'sports':
'https://e00-elmundo.uecdn.es/elmundodeporte/rss/portada.xml',
'technology': '',
'science': 'https://e00-elmundo.uecdn.es/elmundo/rss/ciencia.xml',
'culture': 'https://e00-elmundo.uecdn.es/elmundo/rss/cultura.xml'
},
'el_espanol': { # Esta vacio
'cover_page': '',
'last_news': '',
'spain': '',
'international': '',
'opinion': '',
'sports': '',
'technology': '',
'science': '',
'culture': ''
},
'agencia_efe': { # Sin resumen
'cover_page': 'https://www.efe.com/efe/espana/1/rss'
}
}
def __init__(self):
"""
Init method.
"""
# class variables
self._as = None # SimpleActionServer variable
self._goal = "" # Goal a recibir
self._goal_exec = False # Indicates if goal is being handled
self._pause = False # Indicates if pause
self._step = 'init' # Init, rss_reader, show_info
self._out = False # Varable to exit the while
# Goal varaibles
self._max_time = 0
self._number_plays = 0
self._t0 = 0
self._t1 = 0
self._time_run = 0
self._i_plays = 0
# Local paths
rospack = rospkg.RosPack()
self._root_path = rospack.get_path(pkg_name) # Package path
self._data_path = self._root_path + '/data/' # Data path
self._news_cache = self._data_path + 'news_cache.txt' # Cache file
# Tablet paths
self._default_path = '/default.png' # Default image
# Rss objects
self._feed_name = 'tu_tiempo'
self._category_name = 'cover_page'
#self._rss_feed = self.feeds[self._feed_name][self._category_name] # Feed url
# Cache object
self._cache_manager = CacheManager(self._data_path)
# init the skill
Skill.__init__(self, skill_name, CONDITIONAL)
######################### Skill callbacks #########################
def create_msg_srv(self):
"""
Callback when a start is requested (and skill is stopped).
@raise rospy.ROSException: if the service is inactive.
"""
rospy.logdebug("create_msg_srv() called")
# publishers and subscribers
self.ca_pub = rospy.Publisher("hri_manager/ca_activations",
CA,
queue_size=1) # CA publisher
self.ca_deactivation_pub = rospy.Publisher(
"hri_manager/ca_deactivations", String,
queue_size=1) # CA deactivation publisher
#self.sub_response = rospy.Subscriber(robot + "hri_manager/response", CA, self.response_callback) # CA subscriber
# servers and clients
# Si el servidor actionlib no se ha inicializado:
if not self._as:
self._as = actionlib.SimpleActionServer(skill_name,
news_skill.msg.NewsAction,
execute_cb=self.execute_cb,
auto_start=False)
# start the action server
self._as.start()
def shutdown_msg_srv(self):
"""
Callback when a stop is requested (and skill is running).
"""
# publishers and subscribers
# FIXME: do not unregister publishers because a bug in ROS
rospy.logdebug("shutdown_msg_srv() called")
# Cancel goal
self._as.preempt_request = True
# servers and clients
def pause_exec(self):
"""
Modify the variable self._pause if goal is being handled, when a pause is requested.
"""
if (self._goal_exec): # Goal being handled
self._pause = True
else: # Goal NOT being handled
rospy.logwarn('Goal not being handled')
def resume_exec(self):
"""
Modify the variable self._pause, when a resume is requested.
"""
self._pause = False
self._feedback.app_status = 'resume_ok'
self._as.publish_feedback(self._feedback)
#=================================================================#
def exception_check(self, deactivation=[], t0=-1, t1=-1):
"""
Checks if an exception has been asked. It can be by a preempt request or by a pause request.
@param deactivation: List of CA names to be deactivated (if exception requested)
@param t0: Time 0 to use for time_run (if exception requested)
@param t1: Time 1 to use for time_run (if exception requested)
"""
# Parameters update
if (self._pause or self._as.is_preempt_requested()):
# CA deactivation
if (len(deactivation) > 0):
for msg_name in deactivation:
rospy.logdebug('Deactivating CA: %s' % msg_name)
self.ca_deactivation_pub.publish(msg_name)
# Time update
if (t0 != -1 and t1 != -1):
self._time_run += t1 - t0
# Raise exceptions
############# State Preempted checking #############
# If goal is in Preempted state (that is, there #
# is a goal in the queue or the actual goal is #
# cancelled), the exception is activated. #
####################################################
if (self._as.is_preempt_requested()):
rospy.logwarn("Preempt requested")
raise ActionlibException
return
###################### Pause #######################
if (self._pause):
raise PauseException
return
def pause_wait(self):
"""
Pause loop. Finishes when goal is resumed or cancelled.
"""
rospy.loginfo('Start waiting')
while (self._pause and not self._as.is_preempt_requested()):
rospy.logdebug('waiting...')
rospy.sleep(1)
def get_image(self, article):
"""
Searchs for the best image in the article.
@param article: Article content.
@return image_url: Url of image found. If not found, it returns default image.
@return image_type: Type of image found ('web' or 'image').
"""
image_url, image_type = -1, 'web'
# Search in summary
rospy.logdebug('Searching image in summary')
if 'summary' in article:
soup = BeautifulSoup(article['summary'], features="html.parser")
try:
image_url = soup.find('img')['src']
rospy.logdebug('>> Image selected from summary: %s' %
image_url)
except TypeError as e: # Non existing
image_url = -1
rospy.logdebug('>> No image in summary')
else:
rospy.logwarn('>> No summary')
# Media thumbnail
found = False
rospy.logdebug('Searching image in media thumbnail')
if 'media_thumbnail' in article:
if 'url' in article['media_thumbnail'][0]:
image_url = article['media_thumbnail'][0]['url']
rospy.logdebug('>> Image selected from media_thumbnail: %s' %
image_url)
found = True
if (len(article['media_thumbnail']) > 1):
rospy.logdebug('>> More than 1 media thumbnail: %s' %
len(article['media_thumbnail']))
if (not found):
rospy.logdebug('>> No image in media thumbnail')
# Media content
found = False
rospy.logdebug('Searching image in media content')
if 'media_content' in article:
if 'url' in article['media_content'][0]:
image_url = article['media_content'][0]['url']
rospy.logdebug('>> Image selected from media_content: %s' %
image_url)
found = True
if (len(article['media_content']) > 1):
rospy.logdebug('>> More than 1 media content: %s' %
len(article['media_content']))
if (not found):
rospy.logdebug('>> No image in media content')
# Links
found = False
rospy.logdebug('Searching image in links')
if 'links' in article:
for link in article['links']:
if (link['type'].find('image') >= 0): # It is a image
image_url = link['href']
rospy.logdebug('>> Image selected from links: %s' %
image_url)
found = True
######## Europa Press conversion ########
if (self._feed_name == 'europa_press'):
rospy.logdebug('>> Europa Press conversion')
rospy.logdebug('>> -- Original link: ' + image_url)
# Searchs the position of the extension
pos_l = len(
image_url
) - 5 # Indicates how many character to search from the end of string
pos_r = pos_l + image_url[pos_l:].find(
'.') # Position of the extension ('.jpg')
# Searchs if exists 'vX' at the end
pos_l = pos_r - 4
pos_aux = image_url[pos_l:pos_r].find('_v')
if (pos_aux != -1):
pos_r = pos_l + pos_aux
# Searchs if exists '_XXX' at the end
pos_l = pos_r - 5
pos_aux = image_url[pos_l:pos_r].find('_')
if (pos_aux != -1):
pos_l = pos_l + pos_aux
# Modify the size of the image
s1 = list(image_url)
for i in range(pos_l + 1, pos_r):
s1.pop(pos_l + 1)
s1.insert(pos_l + 1, '800')
image_url = ''.join(s1)
rospy.logdebug('>> -- Modified link: ' + image_url)
##########################################
break
if (not found):
rospy.logdebug('>> No image in links')
if image_url == -1:
return self._default_path, 'image'
else:
return image_url, 'web'
def show_feed_info(self, parsed_content):
"""
Show feed info in terminal.
@param parsed_content: Parsed content of the feed.
"""
print('############### Feed: ###############')
try:
print(parsed_content['feed']['title'])
if (parsed_content['bozo'] == 0):
print 'Xml well-formed'
else:
print 'Xml NOT well-formed'
print('Number of articles: ' + str(len(parsed_content['entries'])))
print('Logo: ' + str(parsed_content['feed']['logo'])
) if 'logo' in parsed_content['feed'] else 'No logo'
print('Image: ' + str(parsed_content['feed']['image']['href'])
) if 'image' in parsed_content['feed'] else 'No image'
except KeyError as e:
rospy.logerr('KeyError: ' + str(e))
print('#####################################')
def show_article_info(self, article):
"""
Show article info in terminal.
@param article: Article content.
@return result: -1 if error found in article, else 0.
"""
if (article == -1):
print('No more articles to show in the category %s' %
self._category_name)
return -1
try:
print('#################### Article: ####################')
###### Title
print('Title:')
print(article['title'])
print('__________________________________________________')
###### Id
print('\nId:')
print(article['id'])
print('__________________________________________________')
###### Summary
print('\nSummary (' + article['summary_detail']['type'] + '):')
# Checks if it is necessary to parse the text
summary_value = article['summary_detail']['value']
if (article['summary_detail']['type'] == 'text/html'):
summary_value = html2text_conv(summary_value)
print(summary_value)
print('##################################################')
except KeyError as e:
rospy.logerr('KeyError: ' + str(e))
return -1
return 0
def show_article_voice(self, article):
"""
Show article info in etts.
@param article: Article content.
@return msg.ca_name: Etts CA name. If error found in article content, returns -1.
"""
rospy.loginfo('Showing article with voice')
if (article == -1):
text = 'No hay mas noticias que mostrar del dia de hoy'
rospy.loginfo('>> %s' % text)
try:
###### Title
title_text = article['title'].encode('utf-8')
rospy.loginfo('>> Title: ' + title_text)
###### Summary
# Checks if it is necessary to parse the text
summary_value = article['summary_detail']['value']
if (article['summary_detail']['type'] == 'text/html'):
summary_value = html2text_conv(summary_value)
rospy.loginfo('>> Summary (%s): ' %
article['summary_detail']['type'])
summary_text = summary_value.encode('utf-8')
rospy.loginfo(summary_text)
text = title_text + ' \\\\pause=1000 ' + summary_text
except KeyError as e:
rospy.logerr('KeyError: ' + str(e))
return -1
msg = makeCA_etts_info(text)
self.ca_pub.publish(msg)
return msg.ca_name
def show_article_tablet(self, image_url, image_type):
"""
Show article info in tablet.
@param image_url: Url of the image to be sent.
@param image_type: Type of the image to be sent.
@return msg.ca_name: Tablet CA name.
"""
rospy.loginfo('Sending image to tablet')
msg = makeCA_tablet_info(image_url, image_type)
self.ca_pub.publish(msg)
return msg.ca_name
def new_article_finder(self, rss_info):
'''
Gets new article. Search in the cache file and if it is not there, it returns the info of the article.
@param rss_info: Parsed content of the feed.
@return article: Article found in the feed. If not found, returns -1. If feed not valid, returns -2.
'''
if (len(rss_info['entries']) == 0):
return -2 # Rss not valid
# Gets cache ids
id_list = self._cache_manager.cache_get_id()
# Loop for the entries
found = False
for article in rss_info['entries']:
found = False
# Checks if id is in the list
for id_n in id_list:
if (article['id'] == id_n):
found = True
break
# If id is not in the list, actual article is selected
if (not found):
return article
break
# All articles have been found
if (found):
return -1
def show_info_handler(self, parsed_content, article, image_url,
image_type):
"""
Show info handler.
@param parsed_content: Parsed content of the feed.
@param article: Article content.
@param image_url: Url of the image to be sent.
@param image_type: Type of the image to be sent.
@return tablet_name_msg: Tablet CA name.
@return etts_name_msg: Etts CA name.
"""
# Feed info
self.show_feed_info(parsed_content)
# Shows the info in the article
self.show_article_info(article)
# Shows image on the tablet
tablet_name_msg = self.show_article_tablet(image_url, image_type)
# Shows info with voice
etts_name_msg = self.show_article_voice(article)
return tablet_name_msg, etts_name_msg
def rss_reader(self):
"""
Rss management function.
@return parsed_content: Parsed content of the feed.
@return article: Article content.
"""
######### Cache refreshing ########
# Refreshes the cache
self._cache_manager.cache_refresh()
###################################
# Search for the next source feed
feed_list = [self._feed_name]
for feed_name in self.feeds:
if feed_name == self._feed_name:
continue
feed_list.append(feed_name)
print feed_list
for feed_name in feed_list:
if self._category_name in self.feeds[feed_name]:
################### Feed found ###################
rospy.loginfo('######## Getting new feed: %s ########' %
feed_name)
self._feed_name = feed_name
feed_url = self.feeds[self._feed_name][self._category_name]
############ Request ############
# Gets the rss feed info
out = False
n_attempts = 0
n_attempts_max = 3
request_ok = False
while (not out):
rospy.logdebug('Number of attempts: %s' % n_attempts)
try:
# Do request using requests library and timeout
rospy.loginfo('Making request')
resp = requests.get(feed_url, timeout=5.05)
except requests.ReadTimeout as e:
rospy.logerr("Timeout when reading RSS: %s" % e)
n_attempts += 1
except requests.exceptions.ConnectionError as e:
rospy.logerr("Connection error; %s" % e)
n_attempts += 1
except requests.exceptions.MissingSchema as e: # URL not valid
rospy.logerr("MissingSchema; %s" % e)
request_ok = False
out = True
else: # No errors
rospy.logdebug('Request done')
request_ok = True
out = True
if (n_attempts >= n_attempts_max):
rospy.logwarn('attempts (%s) >= attempts max (%s)' %
(n_attempts, n_attempts_max))
request_ok = False
out = True
if (not request_ok):
continue
#################################
############ Parsing ############
# Puts it to memory stream object universal feedparser
content = BytesIO(resp.content)
# Parse content
parsed_content = feedparser.parse(content)
#################################
########## Article search #########
# Searchs new article
article = self.new_article_finder(parsed_content)
###################################
######### Article found #########
if (article != -1 and article != -2):
return parsed_content, article
#################################
######### Feed not valid ########
if (article == -2):
rospy.logwarn('Feed not valid')
#################################
####### Article NOT found #######
else:
rospy.logwarn('No more articles in the feed')
#################################
self.show_article_voice(-1)
self.show_article_info(-1)
return -1, -1
def goal_handler(self, goal):
"""
Goal handler.
Checks if the goal is appropriate. If true, it configures the variables for the goal execution.
@param goal: Goal received.
@return goal_accepted: True if goal is accepted, else, False.
"""
# Fill variables
skill_command_vec = goal.skill_command.split('/')
if (len(skill_command_vec) <= 1):
return False
self._feed_name = skill_command_vec[0]
self._category_name = skill_command_vec[1]
self._max_time = goal.max_time
self._number_plays = goal.number_plays
# Check feed name
if not self._feed_name in self.feeds:
rospy.logerr('feed name NOT accepted')
return False
# Check category name
# Gets the name of one feed to get category access
for feed in self.feeds:
feed_name = feed
break
if not self._category_name in self.feeds[feed]:
rospy.logerr('category name NOT accepted')
return False
# Check max_time and number_plays
if (self._max_time > 0 and self._number_plays > 0):
self._limit_method = 'both'
elif (self._max_time > 0):
self._limit_method = 'time'
elif (self._number_plays > 0):
self._limit_method = 'plays'
else:
rospy.logerr(
'max_time and number_plays NOT accepted. Specify one of them')
return False
# Goal accepted
rospy.loginfo('Goal accepted')
return True
def execute_cb(self, goal):
"""
Callback of the node. Activated when a goal is received.
@param goal: news_skill goal.
"""
self._pause = False
self._step = 'Process_goal'
self._limit_method = ''
self._percentage = 0
self._time_run, self._i_plays = 0, 0
self._exec_out = False
percentage_plays, percentage_time = 0, 0
# Result default values
self._result.skill_result = self._result.SUCCESS # Success
# Feedback default values
self._feedback.app_status = 'start_ok'
self._feedback.percentage_completed = 0
self._feedback.engagement = True
self._as.publish_feedback(self._feedback)
############### Si la skill esta activa: ###################
if self._status == self.RUNNING:
self._goal_exec = True # Goal execution starts
print('\n')
rospy.loginfo("RUNNING...")
###################### Exec while ######################
while (not self._exec_out):
try:
rospy.loginfo('Next step: %s' % self._step)
# Wait loop
self.pause_wait() # If wait is asked it enters a loop
self.exception_check(
) # Checks if a exception is requested
# Process_goal
if (self._step == 'Process_goal'):
############ Processes the goal ############
rospy.loginfo('Goal: %s' % goal)
if (not self.goal_handler(goal)): # Goal NOT correct
raise ErrorException('Goal NOT correct')
self._step = 'Get_rss_info'
#==========================================#
self.exception_check(
) # Checks if a exception is requested
t0 = time.time()
# Get_rss_info
if (self._step == 'Get_rss_info'):
# Get rss info
parsed_content, article = self.rss_reader()
if (parsed_content == -1):
raise ErrorException('No More News')
break
self._step = 'Search_image'
self.exception_check(
t0=t0,
t1=time.time()) # Checks if a exception is requested
# Search_image
if (self._step == 'Search_image'):
# Searchs the image in the article
rospy.loginfo('Searching article image')
image_url, image_type = self.get_image(article)
rospy.loginfo('Image selected: %s (%s)' %
(image_url, image_type))
self._step = 'Show_info'
self.exception_check(
t0=t0,
t1=time.time()) # Checks if a exception is requested
# Show_info
if (self._step == 'Show_info'):
# Show info
tablet_name_msg, etts_name_msg = self.show_info_handler(
parsed_content, article, image_url, image_type)
print('Waiting')
i = 0
# Continue when ca has finished
while (i < 10):
self.exception_check(
deactivation=[tablet_name_msg, etts_name_msg],
t0=t0,
t1=time.time(
)) # Checks if a exception is requested
i += 1
rospy.sleep(1)
print('Waited')
self._step = 'Cache_update'
self.exception_check(
t0=t0,
t1=time.time()) # Checks if a exception is requested
# Cache_update
if (self._step == 'Cache_update'):
# Updates the cache with the new article
self._cache_manager.cache_update(article['id'])
# Update limit variables
if (self._limit_method == 'both'
or self._limit_method == 'plays'):
self._i_plays += 1
percentage_plays = int(
(float(self._i_plays) /
float(self._number_plays)) * 100)
rospy.logdebug('percentage plays: %s' %
percentage_plays)
if (self._limit_method == 'both'
or self._limit_method == 'time'):
self._time_run += time.time() - t0
percentage_time = int(
(float(self._time_run) / float(self._max_time))
* 100)
rospy.logdebug('percentage time: %s' %
percentage_time)
self._percentage = percentage_plays if percentage_plays > percentage_time else percentage_time
self._percentage = self._percentage if self._percentage < 100 else 100
rospy.loginfo('percentage: %s' % self._percentage)
self._feedback.percentage_completed = self._percentage
self._step = 'Get_rss_info'
# Exit while
self._exec_out = True if self._feedback.percentage_completed >= 100 else False # Exit if limits are exceeded (number_plays, max_time)
#################### Exceptions ####################
### Preempted or cancel:
except ActionlibException:
rospy.logwarn('[%s] Preempted or cancelled' % pkg_name)
self._exec_out = True
if (self._status == self.STOPPED):
self._feedback.app_status = 'stop_ok'
else:
self._feedback.app_status = 'cancel_ok'
self._result.skill_result = self._result.FAIL # Preempted
### Error
except ErrorException as e:
rospy.logerr(e)
self._exec_out = True
self._result.skill_result = self._result.ERROR # Error
### Pause
except PauseException:
rospy.logwarn('[%s] Paused' % pkg_name)
self._feedback.app_status = 'pause_ok'
self._exec_out = False
#=================== Exceptions ===================#
# Publish feedback at the end of loop
self._as.publish_feedback(self._feedback)
#===================== Exec while =====================#
self._goal_exec = False # Goal execution finished
print('\n')
#==================== Skill activa ========================#
############# Si la skill no esta activa: ##################
else:
rospy.logwarn("STOPPED")
rospy.logwarn("[%s] Cannot send a goal when the skill is stopped" %
pkg_name)
self._result.skill_result = self._result.ERROR # Error
#==========================================================#
#### Envio del resultado y actualizacion del status del goal ###
# Send result
if self._result.skill_result == self._result.SUCCESS:
rospy.logdebug("setting goal to succeeded")
self._feedback.app_status = 'completed_ok'
self._as.publish_feedback(self._feedback)
self._as.set_succeeded(self._result)
else:
rospy.logdebug("setting goal to preempted")
self._feedback.app_status = 'completed_fail'
self._as.publish_feedback(self._feedback)
self._as.set_preempted(self._result)
rospy.loginfo("#############################")
rospy.loginfo("######## Result sent ########")
rospy.loginfo("#############################")
time_milliseconds = time + "{0:03.0f}".format(now.microsecond / 1000)
return u"{}: {}".format(time_milliseconds, msg)
def info(self, msg, *args, **kwargs):
self.logger.info(self.prefix(msg), *args, **kwargs)
def warn(self, msg, *args, **kwargs):
self.logger.warn(self.prefix(msg), *args, **kwargs)
def error(self, msg, *args, **kwargs):
self.logger.error(self.prefix(msg), *args, **kwargs)
def debug(self, msg, *args, **kwargs):
self.logger.debug(self.prefix(msg), *args, **kwargs)
eventLogger = eventLogger('event_consumer_log.log')
cache_client = CacheManager()
def add_item_mylist(item_id, item_value):
"""Add item to mylist
"""
shopping_list = "mylist"
print("Add item to the list:", item_value)
eventLogger.info("Adding item: {} to the:{}".format(item_value, shopping_list))
key = "{}:{}:{}".format(device_id, shopping_list, item_id)
cache_client.set_value(key, str(item_value))
def remove_item(item_id):
"""Remove item to mylist
"""
shopping_list = "mylist"
print("Remove item to the list:", item_id)
def test_get_non_existent_key(self):
"Expected to return None"
cache = CacheManager()
assert not cache.get('idonotexist')
