def cmd_save(server: AppServer, args): if not args: server.clients().save() elif len(args) > 0: if args[0] == "all": server.clients().save() elif args[0] == "server": server.clients().save(False) elif args[0] == "remove": AppData.remove_save() elif args[0] == "help": return CommandReturn(0, "save [server|all|remove]") return CommandReturn(0, "")
def __init__(self, ip="localhost"): RESTServer.__init__(self, ip, HTTPServer.SPAWN_THREAD) self._clients = AppData.load() self._connected = {} self._admins = {} self._clients.get_scheduler().main_loop_in_thread() self.route("POST", "/connect", AppServer.on_connect, self) self.route("GET", "/poll", AppServer.on_poll, self) self.route("GET", "/wait", AppServer.on_wait, self) self.route("PUT", "/file", AppServer.on_put_file, self) self.route("GET", "/file/#id", AppServer.on_get_file, self) self.route("POST", "/result", AppServer.on_result, self) self.route("GET", "/result/#clientid/#cmdid", AppServer.on_get_result, self) self.route("GET", "/admin/login", AppServer.admin_on_auth, self) self.route("GET", "/admin/disconnect", AppServer.admin_on_disconnect, self) self.route("POST", "/admin/command", AppServer.admin_on_command, self) self.route("POST", "/admin/command/texte", AppServer.admin_on_command_texte, self) self.route("POST", "/admin/server/command", AppServer.admin_on_server_command, self) self.route(["GET", "POST"], MOUSTACHES_FILES, AppServer.admin_moustache, self) self.static("/admin", "www", authcb=Callback(AppServer.is_authorized, self), needauthcb=Callback(AppServer.need_auth, self))
def _load_data(self, attempts=3): """Retrieve core data for Nucleus app. Parameters ---------- attempts : int, optional The number of data-loading attempts before raising an error. Each attempt is called 1 second apart. Returns ------- data : AppData Raises ------ StartUpError If data could not be loaded. """ for i in range(attempts): try: data = AppData(Path.DATA_XLSX) except (IOError, EOFError, OSError): if (i + 1) == attempts: raise StartUpError() time.sleep(1) else: return data
print "[Info] Importing rate actions to PredictionIO..." count = 0 for v in app_data.get_rate_actions(): count += 1 if all_info: print "[Info] Importing %s..." % v else: if (count % 32 == 0): sys.stdout.write('\r[Info] %s' % count) sys.stdout.flush() client.identify(v.uid) client.arecord_action_on_item("rate", v.iid, { "pio_rate": v.rating, "pio_t": v.t }) sys.stdout.write('\r[Info] %s rate actions were imported.\n' % count) sys.stdout.flush() if __name__ == '__main__': app_data = AppData() client = predictionio.Client(APP_KEY, THREADS, API_URL, qsize=REQUEST_QSIZE) batch_import_task(app_data, client) client.close()
class App: def __init__(self): self._app_data = AppData() self._client = predictionio.Client(APP_KEY, 1, API_URL) def run(self): state = "[Main Menu]" prompt = "\n"\ "%s\n"\ "%s\n"\ "Please input selection:\n"\ " 0: Quit application.\n"\ " 1: Get Recommendations from PredictionIO.\n"\ " 2: Display user's data." % (state, '-'*len(state)) while True: print prompt choice = raw_input().lower() if choice == '0': print "\nGood Bye!\n" break elif choice == '1': self.recommend_task(state) elif choice == '2': self.display_user_task(state) else: print '[Error] \'%s\' is not a valid selection.' % choice self._client.close() def recommend_task(self, prev_state): state = prev_state + " / [Get Recommendations]" prompt = "\n"\ "%s\n"\ "%s\n"\ "Please enter user id:" % (state, '-'*len(state)) while True: print prompt choice = raw_input().lower() u = self._app_data.get_user(choice) if u: n = 10 print "[Info] Getting top %s item recommendations for user %s..." % (n, u.uid) try: self._client.identify(u.uid) rec = self._client.get_itemrec_topn(ENGINE_NAME, n) u.rec = rec['pio_iids'] self.display_items(u.rec) except predictionio.ItemRecNotFoundError: print "[Info] Recommendation not found" print "[Info] Go back to previous menu..." break else: print "[Error] invalid user id %s. Go back to previous menu..." % choice break def display_user_task(self, prev_state): state = prev_state + " / [Display User]" prompt = "\n"\ "%s\n"\ "%s\n"\ "Please enter user id:" % (state, '-'*len(state)) while True: print prompt choice = raw_input().lower() u = self._app_data.get_user(choice) if u: print "[Info] User %s:" % u.uid n = 10 topn_rate_actions = self._app_data.get_top_rate_actions(u.uid, n) print "\n[Info] Top %s movies rated by this user:"******"\n[Info] Movies recommended to this user:"******"\n[Info] Go back to previous menu..." break else: print "[Error] invalid user id %s. Go back to previous menu..." % choice break def display_items(self, iids, all_info=False): """print item info for each iid in the list """ if iids: for iid in iids: item = self._app_data.get_item(iid) if item: if all_info: print "[Info] %s" % item else: print "[Info] %s" % item.name else: print "[Error] Invalid item id %s" % iid else: print "[Info] Empty." def display_rate_actions(self, actions): """print iid and rating """ if actions: for a in actions: item = self._app_data.get_item(a.iid) if item: print "[Info] %s, rating = %s" % (item.name, a.rating) else: print "[Error] Invalid item id %s" % a.iid else: print "[Info] Empty." def wait_for_ack(self): prompt = "\nPress enter to continue..." print prompt choice = raw_input().lower()
def __init__(self): self._app_data = AppData() self._client = predictionio.Client(APP_KEY, 1, API_URL)
#!/usr/bin/env python3 # -*- coding: utf-8 -*- import tkinter as tk from datetime import datetime import matplotlib.pyplot as plt from appdata import AppData from kpm import KPM if __name__ == "__main__": d = AppData("sample_data.csv") today = datetime(2017, 1, 1) print(d.df) print(d.get_leaderboard_by_date(today)) print(d.get_sums_by_date_range('1/1/2017', '1/7/2017')) plt.figure() d.plot_app_data_in_date_range('Facebook', '1/1/2017', '1/6/2017') d.plot_data_in_date_range('1/1/2017', '1/6/2017') plt.show() dashboard = KPM(d, today) dashboard.mainloop() # print(d.get_means_by_date_range('1/1/2017', '1/4/2017')) # print(d.get_mins_by_date_range('1/1/2017', '1/4/2017')) # print(d.get_maxs_by_date_range('1/1/2017', '1/4/2017')) # print(d.get_app_ranking_by_date('Pinterest', '1/1/2017'))
self.__set(0, 1, self.dt) row_num = 1 for ind in df.index: self.__set(row_num, 0, ind) self.__set(row_num, 1, df[ind]) row_num += 1 def __set(self, row, column, value): widget = self._widgets[row][column] widget.configure(text=value) class TodaysLeaderboard(DailyLeaderboard): def __init__(self, parent, controller, appData, dt): DailyLeaderboard.__init__(self, parent, controller, appData, dt) class ADaysLeaderboard(DailyLeaderboard): def __init__(self, parent, controller, appData, dt): DailyLeaderboard.__init__(self, parent, controller, appData, dt) if __name__ == "__main__": d = AppData("sample_data.csv") print(d.df) dashboard = KPM(d, datetime(2017, 1, 1)) dashboard.mainloop()
class App: def __init__(self): self._app_data = AppData() self._client = predictionio.Client(APP_KEY, 1, API_URL) def run(self): state = "[Main Menu]" prompt = "\n"\ "%s\n"\ "%s\n"\ "Please input selection:\n"\ " 0: Quit application.\n"\ " 1: Get personalized recommendation.\n"\ " 2: Display user data.\n"\ " 3: Display movie data.\n"\ " 4: Recommend with multiple movies.\n" % (state, '-'*len(state)) while True: print prompt choice = raw_input().lower() if choice == '0': print "\nGood Bye!\n" break elif choice == '1': self.recommend_task(state) elif choice == '2': self.display_user_task(state) elif choice == '3': self.get_similar_movies_task(state) elif choice == '4': self.recommend_with_multiple_movies_task(state) else: print '[Error] \'%s\' is not a valid selection.' % choice self._client.close() def recommend_task(self, prev_state): state = prev_state + " / [Get Recommendations]" prompt = "\n"\ "%s\n"\ "%s\n"\ "Please enter user id:" % (state, '-'*len(state)) while True: print prompt choice = raw_input().lower() u = self._app_data.get_user(choice) if u: n = 10 print "[Info] Getting top %s item recommendations for user %s..." % (n, u.uid) try: self._client.identify(u.uid) rec = self._client.get_itemrec_topn(ENGINE_NAME, n) u.rec = rec['pio_iids'] self.display_items(u.rec) except predictionio.ItemRecNotFoundError: print "[Info] Recommendation not found" print "[Info] Go back to previous menu..." break else: print "[Error] invalid user id %s. Go back to previous menu..." % choice break def get_similar_movies_task(self, prev_state): state = prev_state + " / [Get Similar Movies]" prompt = "\n"\ "%s\n"\ "%s\n"\ "Please enter movie id (eg. 1):" % (state, '-'*len(state)) while True: print prompt choice = raw_input().lower() i = self._app_data.get_item(choice) if i: n = 10 self.display_items((i.iid,), all_info=False) print "\n[Info] People who liked this may also liked..." try: rec = self._client.get_itemsim_topn(SIM_ENGINE_NAME, i.iid, n, { 'pio_itypes' : i.genres }) self.display_items(rec['pio_iids'], all_info=False) except predictionio.ItemSimNotFoundError: print "[Info] Similar movies not found" print "[Info] Go back to previous menu..." break else: print "[Error] invalid item id %s. Go back to previous menu..." % choice break def recommend_with_multiple_movies_task(self, prev_state): state = prev_state + " / [Recommend with Multiple Movies]" prompt = "\n"\ "%s\n"\ "%s\n"\ "Please enter comma separated movie ids (eg. 1,2,3):" % (state, '-'*len(state)) while True: print prompt choice = raw_input().lower() viewed_iids = choice.split(",") viewed_items = map(lambda x : self._app_data.get_item(x), viewed_iids) viewed_genres = Set() for i in viewed_items: if i: for g in i.genres: viewed_genres.add(g) if None not in viewed_items: n = 10 self.display_items(viewed_iids, all_info=False) print "\n[Info] Top %s similar movies..." % n try: rec = self._client.get_itemsim_topn(SIM_ENGINE_NAME, choice, n, { 'pio_itypes' : list(viewed_genres) }) self.display_items(rec['pio_iids'], all_info=False) except predictionio.ItemSimNotFoundError: print "[Info] Similar movies not found" print "[Info] Go back to previous menu..." break else: print "[Error] invalid item id %s. Go back to previous menu..." % choice break def display_user_task(self, prev_state): state = prev_state + " / [Display User]" prompt = "\n"\ "%s\n"\ "%s\n"\ "Please enter user id:" % (state, '-'*len(state)) while True: print prompt choice = raw_input().lower() u = self._app_data.get_user(choice) if u: print "[Info] User %s:" % u.uid n = 10 topn_rate_actions = self._app_data.get_top_rate_actions(u.uid, n) print "\n[Info] Top %s movies rated by this user:"******"\n[Info] Getting New Recommendation..." n = 10 try: self._client.identify(u.uid) rec = self._client.get_itemrec_topn(ENGINE_NAME, n) u.rec = rec['pio_iids'] except predictionio.ItemRecNotFoundError: print "[Info] Recommendation not found" print "\n[Info] Movies recommended to this user:"******"\n[Info] Go back to previous menu..." break else: print "[Error] invalid user id %s. Go back to previous menu..." % choice break def display_items(self, iids, all_info=False): """print item info for each iid in the list """ if iids: for iid in iids: item = self._app_data.get_item(iid) if item: if all_info: print "[Info] %s" % item else: print "[Info] (%s) %s %s %s" % (item.iid, item.name, item.release_date.strftime("%d-%b-%Y"), item.genres) else: print "[Error] Invalid item id %s" % iid else: print "[Info] Empty." def display_rate_actions(self, actions): """print iid and rating """ if actions: for a in actions: item = self._app_data.get_item(a.iid) if item: print "[Info] (%s) %s %s %s, rating = %s" % (item.iid, item.name, item.release_date.strftime("%d-%b-%Y"), item.genres, a.rating) else: print "[Error] Invalid item id %s" % a.iid else: print "[Info] Empty." def wait_for_ack(self): prompt = "\nPress enter to continue..." print prompt choice = raw_input().lower()
def build(self, filename): # check the package has .desktop files print 'SOURCE\t', filename pkg = Package(filename) for b in self.cfg.get_package_blacklist(): if fnmatch.fnmatch(pkg.name, b): print 'IGNORE\t', filename, '\t', "package is blacklisted:", pkg.name return # set up state if not os.path.exists('./appstream'): os.makedirs('./appstream') if not os.path.exists('./icons'): os.makedirs('./icons') if not os.path.exists('./screenshot-cache'): os.makedirs('./screenshot-cache') if not os.path.exists('./screenshots'): os.makedirs('./screenshots') if not os.path.exists('./screenshots/source'): os.makedirs('./screenshots/source') for size in self.cfg.get_screenshot_thumbnail_sizes(): path = './screenshots/' + str(size[0]) + 'x' + str(size[1]) if not os.path.exists(path): os.makedirs(path) # remove tmp if os.path.exists('./tmp'): shutil.rmtree('./tmp') os.makedirs('./tmp') # decompress main file and search for desktop files package_decompress(pkg) files = [] for f in self.cfg.get_interesting_installed_files(): files.extend(glob.glob("./tmp" + f)) files.sort() # we only need to install additional files if we're not running on # the builders for c in self.cfg.get_package_data_list(): if fnmatch.fnmatch(pkg.name, c[0]): extra_files = glob.glob("./packages/%s*.rpm" % c[1]) for f in extra_files: extra_pkg = Package(f) print "INFO\tAdding extra package %s for %s" % (extra_pkg.name, pkg.name) package_decompress(extra_pkg) # open the AppStream file for writing xml_output_file = './appstream/' + pkg.name + '.xml' xml = open(xml_output_file, 'w') xml.write("<?xml version=\"1.0\"?>\n") xml.write("<applications version=\"0.1\">\n") # check for duplicate apps in the package application_ids = [] has_valid_content = False # process each desktop file in the original package for f in files: print 'PROCESS\t', f fi = Gio.file_new_for_path(f) info = fi.query_info('standard::content-type', 0, None) # create the right object depending on the content type content_type = info.get_content_type() if content_type == 'inode/symlink': continue if content_type == 'application/x-font-ttf': app = FontFile(pkg, self.cfg) elif content_type == 'application/x-font-otf': app = FontFile(pkg, self.cfg) elif content_type == 'application/x-desktop': app = DesktopFile(pkg, self.cfg) elif content_type == 'application/xml': app = InputMethodComponent(pkg, self.cfg) elif content_type == 'application/x-sqlite3': app = InputMethodTable(pkg, self.cfg) else: print 'IGNORE\t', f, '\t', "content type " + content_type + " not supported" continue # the ID is the filename app.set_id(f.split('/')[-1]) # application is blacklisted blacklisted = False for b in self.cfg.get_id_blacklist(): if fnmatch.fnmatch(app.app_id, b): print 'IGNORE\t', f, '\t', "application is blacklisted:", app.app_id blacklisted = True break if blacklisted: continue # packages that ship .desktop files in /usr/share/applications # *and* /usr/share/applications/kde4 do not need multiple entries if app.app_id in application_ids: print 'IGNORE\t', f, '\t', app.app_id, 'duplicate ID in package' continue application_ids.append(app.app_id) # parse desktop file if not app.parse_file(f): continue # do we have an AppData file? appdata_file = './tmp/usr/share/appdata/' + app.app_id + '.appdata.xml' appdata_extra_file = './appdata-extra/' + app.type_id + '/' + app.app_id + '.appdata.xml' if os.path.exists(appdata_file) and os.path.exists(appdata_extra_file): print 'DELETE\t', appdata_extra_file, 'as upstream AppData file exists' os.remove(appdata_extra_file) # just use the extra file in places of the missing upstream one if os.path.exists(appdata_extra_file): appdata_file = appdata_extra_file # need to extract details if os.path.exists(appdata_file): data = AppData() data.extract(appdata_file) # check AppData file validates if os.path.exists('/usr/bin/appdata-validate'): env = os.environ p = subprocess.Popen(['/usr/bin/appdata-validate', '--relax', appdata_file], cwd='.', env=env, stdout=subprocess.PIPE) p.wait() if p.returncode: for line in p.stdout: line = line.replace('\n', '') print 'WARNING\tAppData did not validate: ' + line # check the id matches if data.get_id() != app.app_id and data.get_id() != app.app_id_full: raise StandardError('The AppData id does not match: ' + app.app_id) # check the licence is okay if data.get_licence() not in self.cfg.get_content_licences(): raise StandardError('The AppData licence is not okay for ' + app.app_id + ': \'' + data.get_licence() + '\'') # if we have an override, use it for all languages tmp = data.get_names() if tmp: app.names = tmp # if we have an override, use it for all languages tmp = data.get_summaries() if tmp: app.comments = tmp # get optional bits tmp = data.get_url() if tmp: app.homepage_url = tmp tmp = data.get_project_group() if tmp: app.project_group = tmp app.descriptions = data.get_descriptions() # get screenshots tmp = data.get_screenshots() for image in tmp: print 'DOWNLOADING\t', image app.add_screenshot_url(image) elif app.requires_appdata: print 'IGNORE\t', f, '\t', app.app_id_full, 'requires AppData to be included' continue # use the homepage to filter out same more generic apps if not app.project_group: # GNOME project_urls = [ 'http*://*.gnome.org*', 'http://gnome-*.sourceforge.net/'] for m in project_urls: if fnmatch.fnmatch(app.homepage_url, m): app.project_group = "GNOME" # KDE project_urls = [ 'http*://*.kde.org*', 'http://*kde-apps.org/*' ] for m in project_urls: if fnmatch.fnmatch(app.homepage_url, m): app.project_group = "KDE" # XFCE project_urls = [ 'http://*xfce.org*' ] for m in project_urls: if fnmatch.fnmatch(app.homepage_url, m): app.project_group = "XFCE" # LXDE project_urls = [ 'http://lxde.org*', 'http://lxde.sourceforge.net/*' ] for m in project_urls: if fnmatch.fnmatch(app.homepage_url, m): app.project_group = "LXDE" # MATE project_urls = [ 'http://*mate-desktop.org*' ] for m in project_urls: if fnmatch.fnmatch(app.homepage_url, m): app.project_group = "MATE" # print that we auto-added it if app.project_group: print 'INFO\t', f, '\t', app.app_id, 'assigned', app.project_group # we got something useful if not has_valid_content: has_valid_content = True # Do not include apps without a name if not 'C' in app.names: print 'IGNORE\t', f, '\t', "no Name" continue # Do not include apps without a summary if not 'C' in app.comments: print 'IGNORE\t', f, '\t', "no Comment" continue # Do not include apps without an icon if not app.icon: print 'IGNORE\t', f, '\t', "Icon unspecified" continue # write content app.write(xml) # create AppStream XML xml.write("</applications>\n") xml.close() if not has_valid_content: os.remove(xml_output_file) # create AppStream icon tar if has_valid_content: output_file = "./appstream/%s-icons.tar" % pkg.name print 'WRITING\t', output_file tar = tarfile.open(output_file, "w") files = glob.glob("./icons/*.png") for f in files: tar.add(f, arcname=f.split('/')[-1]) tar.close() # remove tmp if not os.getenv('APPSTREAM_DEBUG'): shutil.rmtree('./tmp') shutil.rmtree('./icons')
def batch(recon_root, proc_recon_root, csv_path): """ Script to automatically generate crops, scled images and compressed files :param recon_list: :param mount: :return: """ with open(csv_path, 'r') as fh: recon_list = [row[0] for row in csv.reader(fh)] app = AppData() auto = Autofill(None) update = FakeUpdate() for recon_id in recon_list: stage = get_stage(recon_id) proc_recon_path = join(proc_recon_root, recon_id) cropped_dir = join(proc_recon_path, 'cropped') if not isdir(cropped_dir): mkdir(cropped_dir) scaled_dir = join(proc_recon_path, 'scaled_stacks') metadata_dir = join(proc_recon_path, 'Metadata') recon_id = get_input_id(join(metadata_dir, 'config4user.log')) recon_path = join(recon_root, recon_id) # Performing cropping if directory does not exist or is empty if len(listdir(cropped_dir)) == 0: fake_config = lambda: None fake_config.meta_path = metadata_dir fake_config.value = 0 cropper = Crop(recon_path, cropped_dir, update.emit, fake_config, fake_config, app, def_crop=None, repeat_crop=None) img_list = cropper.run(auto=True) else: img_list = app.getfilelist(cropped_dir) # Get recon log and pixel size log_paths = [f for f in listdir(cropped_dir) if f.endswith("_rec.log")] if len(log_paths) < 1: print('Cannot find log in cropped directory') continue log = join(cropped_dir, log_paths[0]) with open(log, 'rb') as log_file: original_pixel_size = float(auto.get_pixel(stage, log_file)) # Scaling if not isdir(scaled_dir): mkdir(scaled_dir) for scale in SCALING[stage]: scale_by_int = False if type(scale) is int: sf = scale new_pixel_size = original_pixel_size * float(scale) scale_by_int = True else: sf = float(scale) / float(original_pixel_size) new_pixel_size = sf * original_pixel_size out_name = join( scaled_dir, '{}_scaled_{:.4f}_pixel_{:.2f}.{}'.format( recon_id, sf, new_pixel_size, ext)) if scaled_stack_exists(scaled_dir, sf, new_pixel_size): continue resample(img_list, sf, out_name, scale_by_int, update) # Compression bz2_file = join(proc_recon_path, 'IMPC_cropped_{}.nrrd'.format(recon_id)) if not isfile(bz2_file + '.bz2'): print "Generating missing bz2 file for '{}'".format(recon_id) try: bz2_nnrd(img_list, bz2_file, 'Compressing cropped recon', update) except IOError as e: print( 'Failed to write the compressed bzp2 file. Network issues?\n{}' .format(e))
def add_appdata_file(self): # do we have an AppData file? filename = './tmp/usr/share/appdata/' + self.app_id + '.appdata.xml' fn_extra = '../appdata-extra/' + self.type_id + '/' + self.app_id + '.appdata.xml' if os.path.exists(filename) and os.path.exists(fn_extra): self.log.write(LoggerItem.INFO, "deleting %s as upstream AppData file exists" % fn_extra) os.remove(fn_extra) # just use the extra file in places of the missing upstream one if os.path.exists(fn_extra): filename = fn_extra # need to extract details if not os.path.exists(filename): return False data = AppData() if not data.extract(filename): self.log.write(LoggerItem.WARNING, "AppData file '%s' could not be parsed" % filename) return False # check AppData file validates enable_validation = self.type_id != 'font' if enable_validation and os.path.exists('/usr/bin/appdata-validate'): env = os.environ p = subprocess.Popen(['/usr/bin/appdata-validate', '--relax', filename], cwd='.', env=env, stdout=subprocess.PIPE) p.wait() if p.returncode: for line in p.stdout: line = line.replace('\n', '').decode('utf-8') self.log.write(LoggerItem.WARNING, "AppData did not validate: %s" % line) # check the id matches if data.get_id() != self.app_id and data.get_id() != self.app_id_full: self.log.write(LoggerItem.WARNING, "The AppData id does not match: " + self.app_id) return False # check the licence is okay if data.get_licence() not in self.cfg.get_content_licences(): self.log.write(LoggerItem.WARNING, "The AppData licence is not okay for " + self.app_id + ': \'' + data.get_licence() + '\'') return False # if we have an override, use it for all languages tmp = data.get_names() if tmp: self.names = tmp # if we have an override, use it for all languages tmp = data.get_summaries() if tmp: self.comments = tmp # get metadata tmp = data.get_metadata() if tmp: # and extra packages we want to add in? if 'ExtraPackages' in tmp: for pkg in tmp['ExtraPackages'].split(','): if pkg not in self.pkgnames: self.pkgnames.append(pkg) del tmp['ExtraPackages'] self.metadata.update(tmp) # get optional bits tmp = data.get_urls() if tmp: for key in tmp: self.urls[key] = tmp[key] tmp = data.get_project_group() if tmp: self.project_group = tmp try: self.descriptions = data.get_descriptions() except StandardError, e: self.log.write(LoggerItem.WARNING, "failed to add description: %s" % str(e))