def import_snippets(self, filenames): success = True for filename in filenames: if not Pluma.utils_uri_has_file_scheme(filename): continue # Remove file:// gfile = Gio.file_new_for_uri(filename) filename = gfile.get_path() importer = Importer(filename) error = importer.run() if error: message = _( 'The following error occurred while importing: %s') % error success = False message_dialog(self.dlg, Gtk.MessageType.ERROR, message) self.build_model(True) if success: message = _('Import successfully completed') message_dialog(self.dlg, Gtk.MessageType.INFO, message)
def _reload(self, filenames): modules = [m for m in sys.modules.values() if self._check(filenames, m)] for mod in modules: self._logger.info("Reloading module %s", mod.__name__) Importer.reload(mod) else: j25._load_routing() j25._update_mapper() j25._dispatcher.register_all_apps_router()
def __init__(self, settings, config, sql=None, parent=None, cli=False): self.importtimer = 0 self.settings = settings self.config = config self.sql = sql self.parent = parent self.input_settings = {} self.pipe_to_hud = None self.importer = Importer.Importer(self, self.settings, self.config, self.sql) self.importer.setCallHud(True) self.importer.setQuiet(False) self.importer.setHandCount(0) self.importer.setMode('auto') self.server = settings['db-host'] self.user = settings['db-user'] self.password = settings['db-password'] self.database = settings['db-databaseName'] if cli == False: self.setupGui() else: # TODO: Separate the code that grabs the directories from config # Separate the calls to the Importer API # Create a timer interface that doesn't rely on GTK pass
def __init__(self): gobject.GObject.__init__(self) self.conf = Config.Config(tgcm.country_support) self.modem_manager = MainModem.MainModem() self.conn_manager = ConnectionManager.ConnectionManager() self.device_dialer = FreeDesktop.DeviceDialer() self.importer = Importer.Importer() self.hs_service = HotSpotsService.HotSpotsService() self.install_callback = None self.refresh_callback = None if not self.conf.is_news_available(): return # Load news and updates database self.db_name = 'news-%s.db' % tgcm.country_support self.db_filepath = os.path.join(tgcm.config_dir, self.db_name) if os.path.exists(self.db_filepath): db_file = open(self.db_filepath, 'r') self.db_contents = pickle.load(db_file) db_file.close() else: self.db_contents = {} self.db_contents['news'] = {} self.db_contents['updates'] = {} self.__update_rss_db_if_possible(APP_START) self.device_dialer.connect("connected", self.__connected_cb) self.modem_manager.connect('main-modem-changed', self.__main_modem_changed_cb) self.modem_manager.connect('main-modem-removed', self.__main_modem_removed_cb)
def send(): """Renders the main page of the US Parks Finder Website""" if request.method == 'POST': name = request.form['Name'] state = request.form['State'] loc = request.form['Location'] mtd = int(request.form['maxTravelDistance']) user = MDL.User(name, state, loc, float(mtd)) usrLat = user.lat usrLng = user.lng parks = IMPRT.getParks(user) #print(parks) #parks.sort() #print(parks) zoom = getZoom(mtd) pNames = getPNames(parks) pNames.sort() pLa, pLo = getPLatLngs(parks) pTups = getPTups(parks) return render_template('test.html', name=name, state=state, location=loc, tdist=mtd, numParks=len(pNames), parks=pNames, usrLat=usrLat, usrLng=usrLng, zoom=zoom, pLa=pLa, pLo=pLo, parkObjs=pTups) else: return render_template("index.html")
def __init__(self, settings, config, sql=None, parent=None): QWidget.__init__(self, parent) self.settings = settings self.config = config self.importer = Importer.Importer(self, self.settings, config, sql, self) self.setLayout(QVBoxLayout()) self.importDir = QLineEdit(self.settings['bulkImport-defaultPath']) hbox = QHBoxLayout() hbox.addWidget(self.importDir) self.chooseButton = QPushButton('Browse...') self.chooseButton.clicked.connect(self.browseClicked) hbox.addWidget(self.chooseButton) self.layout().addLayout(hbox) self.load_button = QPushButton(_('Bulk Import')) self.load_button.clicked.connect(self.load_clicked) self.layout().addWidget(self.load_button) # see how many hands are in the db and adjust accordingly tcursor = self.importer.database.cursor tcursor.execute("Select count(1) from Hands") row = tcursor.fetchone() tcursor.close() self.importer.database.rollback() self.n_hands_in_db = row[0]
def getMoreParkData(pName, usrLatLng): """returns in depth data about one park for the more page -inputs: pName - type:string, usrLatLng - type:tuple """ park = IMPRT.getPark(pName) park.setWeatherAndTemp(run=True) park.getDistance(latlng=usrLatLng) return park
def __init__(self, settings, config, sql = None, parent = None): self.settings = settings self.config = config self.parent = parent self.importer = Importer.Importer(self, self.settings, config, sql, parent) self.vbox = gtk.VBox(False, 0) self.vbox.show() self.chooser = gtk.FileChooserWidget() self.chooser.set_filename(self.settings['bulkImport-defaultPath']) self.chooser.set_select_multiple(True) self.chooser.set_show_hidden(True) self.vbox.add(self.chooser) self.chooser.show() # Table widget to hold the progress bar and load button #self.table = gtk.Table(rows=5, columns=5, homogeneous=False) self.table = gtk.Table(rows=2, columns=4, homogeneous=False) self.vbox.add(self.table) self.table.show() # label - spacer (fills in column one in table) self.lab_spacer = gtk.Label() # self.table.attach(self.lab_spacer, 3, 5, 3, 4, xpadding=0, ypadding=0, # yoptions=gtk.SHRINK) self.table.attach(self.lab_spacer,1,2,1,2, xpadding=0, ypadding=0, yoptions=gtk.SHRINK) self.lab_spacer.show() # button - Import self.load_button = gtk.Button(_('_Bulk Import')) # todo: rename variables to import too self.load_button.connect('clicked', self.load_clicked, _('Import clicked')) # self.table.attach(self.load_button, 2, 3, 4, 5, xpadding=0, ypadding=0, # yoptions=gtk.SHRINK) self.table.attach(self.load_button, 2, 3, 1, 2, xpadding=0, ypadding=0, yoptions=gtk.SHRINK) self.load_button.show() # label - info self.progressbar = gtk.ProgressBar() self.table.attach(self.progressbar, 3, 4, 1, 2, xpadding=0, ypadding=0, yoptions=gtk.SHRINK) self.progressbar.set_text(_("Waiting...")) self.progressbar.set_fraction(0) self.progressbar.show() # see how many hands are in the db and adjust accordingly tcursor = self.importer.database.cursor tcursor.execute("Select count(1) from Hands") row = tcursor.fetchone() tcursor.close() self.importer.database.rollback() self.n_hands_in_db = row[0]
def import_activity(import_str): local_file_name = "" try: import_obj = json.loads(import_str) username = import_obj['username'] user_id = import_obj['user_id'] uploaded_file_data = import_obj['uploaded_file_data'] uploaded_file_name = import_obj['uploaded_file_name'] # Generate a random name for the local file. print("Generating local file name...") root_dir = os.path.dirname(os.path.abspath(__file__)) tempfile_dir = os.path.join(root_dir, 'tempfile') if not os.path.exists(tempfile_dir): os.makedirs(tempfile_dir) upload_path = os.path.normpath(tempfile_dir) uploaded_file_name, uploaded_file_ext = os.path.splitext( uploaded_file_name) local_file_name = os.path.join(upload_path, str(uuid.uuid4())) local_file_name = local_file_name + uploaded_file_ext # Write the file. print("Write the data to a local file...") with open(local_file_name, 'wb') as local_file: local_file.write(uploaded_file_data.encode("utf-8")) # Import the file into the database. print("Import the data to the database...") data_mgr = DataMgr.DataMgr("", None, None, None) importer = Importer.Importer(data_mgr) success, _, activity_id = importer.import_file(username, user_id, local_file_name, uploaded_file_name, uploaded_file_ext) # If the import was successful, then schedule the activity for analysis. if success: print("Import was successful, perform analysis...") analysis_scheduler = AnalysisScheduler.AnalysisScheduler() activity = data_mgr.retrieve_activity(activity_id) analysis_scheduler.add_to_queue(activity, user_id, data_mgr) else: print("Import was not successful.") except: log_error("Exception when importing activity data: " + str(import_str)) log_error(traceback.format_exc()) log_error(sys.exc_info()[0]) finally: # Remove the local file. if len(local_file_name) > 0: print("Removing local file...") os.remove(local_file_name)
def wacot_import(args): importer = Importer.Importer() if args.from_dumps == 'all' or args.from_dumps == 'xml': importer.import_xml() if args.from_dumps == 'all' or args.from_dumps == 'cat': importer.import_categories() if not args.only_import: analyzer = Analyzer.Analyzer() analyzer.compute_article_contributions() analyzer.compute_category_contributions() analyzer.compute_bot_flags() analyzer.count_article_contributions() analyzer.count_category_contributions()
def upload(): """ Imports a file and saves to DB :return: """ datafile = request.files['file'] c = MySQL.get_connection(DATABASE) if datafile: try: logfile = os.path.splitext(datafile.filename)[0] + str( int(time.time())) + '.log' # given name + current timestamp f = logging.FileHandler(os.path.join(LOG_DIR, logfile), 'w') Config.setup_logging(f) filepath = os.path.join(UPLOADS_DIR, datafile.filename) datafile.save(filepath) # to file system Importer.run(filepath, c, {"normalization": request.form['normalization']}) logger.removeHandler(f) f.close() return jsonify({"name": datafile.filename, 'log': logfile}) finally: c.close()
def import_snippets(self, filenames): success = True for filename in filenames: if not gedit.utils.uri_has_file_scheme(filename): continue # Remove file:// gfile = gio.File(filename) filename = gfile.get_path() importer = Importer(filename) error = importer.run() if error: message = _('The following error occurred while importing: %s') % error success = False message_dialog(self.dlg, gtk.MESSAGE_ERROR, message) self.build_model(True) if success: message = _('Import successfully completed') message_dialog(self.dlg, gtk.MESSAGE_INFO, message)
def main(argv=None): """main can also be called in the python interpreter, by supplying the command line as the argument.""" if argv is None: argv = sys.argv[1:] def destroy(*args): # call back for terminating the main eventloop gtk.main_quit() Configuration.set_logfile("fpdb-log.txt") (options, argv) = Options.fpdb_options() if options.sitename: options.sitename = Options.site_alias(options.sitename) if options.usage == True: #Print usage examples and exit print _("USAGE:") sys.exit(0) Configuration.set_logfile("GuiBulkImport-log.txt") if options.config: config = Configuration.Config(options.config) else: config = Configuration.Config() settings = {} if os.name == 'nt': settings['os'] = 'windows' else: settings['os'] = 'linuxmac' settings.update(config.get_db_parameters()) settings.update(config.get_import_parameters()) settings.update(config.get_default_paths()) #Do something useful importer = Importer.Importer(False, settings, config, None) importer.addBulkImportImportFileOrDir(os.path.expanduser(options.filename)) importer.setCallHud(False) if options.archive: importer.setStarsArchive(True) importer.setFTPArchive(True) if options.testData: importer.setPrintTestData(True) (stored, dups, partial, errs, ttime) = importer.runImport() importer.clearFileList() print(_('Bulk import done: Stored: %d, Duplicates: %d, Partial: %d, Errors: %d, Time: %s seconds, Stored/second: %.0f')\ % (stored, dups, partial, errs, ttime, (stored+0.0) / ttime))
class Parser(object): def __init__(self): self.objects = {} self.importer = Importer() def _import(self, file_path): getFileName = lambda path: splitext(split(file_path)[1])[0] with open(file_path) as file: data = self.importer.do_import(file) return data, getFileName(file_path) def _add_obj(self, data, file_name): """Regista o objeto no parser""" vertices, normals, faces, textures = data self.objects[file_name] = { 'vertices': vertices, 'normals': normals, 'faces': faces, 'textures': textures, } def _parse(self, name, use_texture : bool): obj = self.objects[name] glBegin(GL_QUADS) for face in obj['faces']: # estrutura de um vertice na lista de # faces: vertex/texture/normal for v in face: vi, ti, ni = v glNormal3fv(obj['normals'][ni]) glTexCoord2fv(obj['textures'][ti]) glVertex3fv(obj['vertices'][vi]) glEnd() def load(self, file_path): """Carrega o arquivo""" data, file_name = self._import(file_path) self._add_obj(data, file_name) def parse(self, name, texture=False): """Faz o desenho do objeto""" self._parse(name, texture)
import Importer Importer.loadImports('data/blueprints/') from data.blueprints import * class blueprint: def __init__(self, ID, width, height, tiles): self.ID = ID self.tiles = tiles self.width = width self.height = height def getRel(self, colRow): toRet = [] for t in range(len(self.tiles)): toRet.append( (colRow[1] - self.tiles[t][0], self.tiles[t][1] + colRow[0], self.tiles[t][2], self.tiles[t][3])) return toRet def getRequiredResources(self): req = [] for t in range(len(self.tiles)): found = False for i in range(len(req)): if req[i][0] == self.tiles[t][3]: found = True
def setUp(self): self.imp = Importer()
def __init__(self, settings, config, sql=None, parent=None): self.settings = settings self.config = config self.parent = parent self.importer = Importer.Importer(self, self.settings, config, sql, parent) self.vbox = gtk.VBox(False, 0) self.vbox.show() self.chooser = gtk.FileChooserWidget() self.chooser.set_filename(self.settings['bulkImport-defaultPath']) self.chooser.set_select_multiple(True) self.chooser.set_show_hidden(True) self.vbox.add(self.chooser) self.chooser.show() # Table widget to hold the settings self.table = gtk.Table(rows=5, columns=5, homogeneous=False) self.vbox.add(self.table) self.table.show() # checkbox - print start/stop? self.chk_st_st = gtk.CheckButton(_('Print Start/Stop Info')) self.table.attach(self.chk_st_st, 0, 1, 0, 1, xpadding=10, ypadding=0, yoptions=gtk.SHRINK) self.chk_st_st.show() self.chk_st_st.set_active(True) # label - status self.lab_status = gtk.Label(_("Hands/status print:")) self.table.attach(self.lab_status, 1, 2, 0, 1, xpadding=0, ypadding=0, yoptions=gtk.SHRINK) self.lab_status.show() self.lab_status.set_justify(gtk.JUSTIFY_RIGHT) self.lab_status.set_alignment(1.0, 0.5) # spin button - status status_adj = gtk.Adjustment( value=100, lower=0, upper=300, step_incr=10, page_incr=1, page_size=0) #not sure what upper value should be! self.spin_status = gtk.SpinButton(adjustment=status_adj, climb_rate=0.0, digits=0) self.table.attach(self.spin_status, 2, 3, 0, 1, xpadding=10, ypadding=0, yoptions=gtk.SHRINK) self.spin_status.show() # label - threads self.lab_threads = gtk.Label(_("Number of threads:")) self.table.attach(self.lab_threads, 3, 4, 0, 1, xpadding=0, ypadding=0, yoptions=gtk.SHRINK) self.lab_threads.show() if not self.allowThreads: self.lab_threads.set_sensitive(False) self.lab_threads.set_justify(gtk.JUSTIFY_RIGHT) self.lab_threads.set_alignment(1.0, 0.5) # spin button - threads threads_adj = gtk.Adjustment( value=0, lower=0, upper=32, step_incr=1, page_incr=1, page_size=0) #not sure what upper value should be! self.spin_threads = gtk.SpinButton(adjustment=threads_adj, climb_rate=0.0, digits=0) self.table.attach(self.spin_threads, 4, 5, 0, 1, xpadding=10, ypadding=0, yoptions=gtk.SHRINK) self.spin_threads.show() if not self.allowThreads: self.spin_threads.set_sensitive(False) # checkbox - archive file? self.is_archive = gtk.CheckButton(_('Archive File')) self.table.attach(self.is_archive, 0, 1, 1, 2, xpadding=10, ypadding=0, yoptions=gtk.SHRINK) self.is_archive.show() # label - hands self.lab_hands = gtk.Label(_("Hands/file:")) self.table.attach(self.lab_hands, 1, 2, 1, 2, xpadding=0, ypadding=0, yoptions=gtk.SHRINK) self.lab_hands.show() self.lab_hands.set_justify(gtk.JUSTIFY_RIGHT) self.lab_hands.set_alignment(1.0, 0.5) # spin button - hands to import hands_adj = gtk.Adjustment( value=0, lower=0, upper=10, step_incr=1, page_incr=1, page_size=0) #not sure what upper value should be! self.spin_hands = gtk.SpinButton(adjustment=hands_adj, climb_rate=0.0, digits=0) self.table.attach(self.spin_hands, 2, 3, 1, 2, xpadding=10, ypadding=0, yoptions=gtk.SHRINK) self.spin_hands.show() # label - drop indexes self.lab_drop = gtk.Label(_("Drop indexes:")) self.table.attach(self.lab_drop, 3, 4, 1, 2, xpadding=0, ypadding=0, yoptions=gtk.SHRINK) self.lab_drop.show() self.lab_drop.set_justify(gtk.JUSTIFY_RIGHT) self.lab_drop.set_alignment(1.0, 0.5) # ComboBox - drop indexes self.cb_dropindexes = gtk.combo_box_new_text() self.cb_dropindexes.append_text(_('auto')) self.cb_dropindexes.append_text(_("don't drop")) self.cb_dropindexes.append_text(_('drop')) self.cb_dropindexes.set_active(0) self.table.attach(self.cb_dropindexes, 4, 5, 1, 2, xpadding=10, ypadding=0, yoptions=gtk.SHRINK) self.cb_dropindexes.show() self.cb_testmode = gtk.CheckButton(_('HUD Test mode')) self.table.attach(self.cb_testmode, 0, 1, 2, 3, xpadding=10, ypadding=0, yoptions=gtk.SHRINK) self.cb_testmode.show() # label - filter self.lab_filter = gtk.Label(_("Site filter:")) self.table.attach(self.lab_filter, 1, 2, 2, 3, xpadding=0, ypadding=0, yoptions=gtk.SHRINK) self.lab_filter.show() self.lab_filter.set_justify(gtk.JUSTIFY_RIGHT) self.lab_filter.set_alignment(1.0, 0.5) # ComboBox - filter self.cbfilter = gtk.combo_box_new_text() disabled_sites = [] # move disabled sites to bottom of list self.cbfilter.append_text(_("Please select site")) for w in self.config.hhcs: try: if self.config.supported_sites[ w].enabled: # include enabled ones first print w self.cbfilter.append_text(w) else: disabled_sites.append(w) except: # self.supported_sites[w] may not exist if hud_config is bad disabled_sites.append(w) for w in disabled_sites: # then disabled ones print w self.cbfilter.append_text(w) self.cbfilter.set_active(0) self.table.attach(self.cbfilter, 2, 3, 2, 3, xpadding=10, ypadding=1, yoptions=gtk.SHRINK) self.cbfilter.show() # label - drop hudcache self.lab_hdrop = gtk.Label(_("Drop HudCache:")) self.table.attach(self.lab_hdrop, 3, 4, 2, 3, xpadding=0, ypadding=0, yoptions=gtk.SHRINK) self.lab_hdrop.show() self.lab_hdrop.set_justify(gtk.JUSTIFY_RIGHT) self.lab_hdrop.set_alignment(1.0, 0.5) # ComboBox - drop hudcache self.cb_drophudcache = gtk.combo_box_new_text() self.cb_drophudcache.append_text(_('auto')) self.cb_drophudcache.append_text(_("don't drop")) self.cb_drophudcache.append_text(_('drop')) self.cb_drophudcache.set_active(0) self.table.attach(self.cb_drophudcache, 4, 5, 2, 3, xpadding=10, ypadding=0, yoptions=gtk.SHRINK) self.cb_drophudcache.show() # button - Import self.load_button = gtk.Button( _('_Bulk Import')) # todo: rename variables to import too self.load_button.connect('clicked', self.load_clicked, _('Import clicked')) self.table.attach(self.load_button, 2, 3, 4, 5, xpadding=0, ypadding=0, yoptions=gtk.SHRINK) self.load_button.show() # label - spacer (keeps rows 3 & 5 apart) self.lab_spacer = gtk.Label() self.table.attach(self.lab_spacer, 3, 5, 3, 4, xpadding=0, ypadding=0, yoptions=gtk.SHRINK) self.lab_spacer.show() # label - info # self.lab_info = gtk.Label() # self.table.attach(self.lab_info, 3, 5, 4, 5, xpadding = 0, ypadding = 0, yoptions=gtk.SHRINK) # self.lab_info.show() self.progressbar = gtk.ProgressBar() self.table.attach(self.progressbar, 3, 5, 4, 5, xpadding=0, ypadding=0, yoptions=gtk.SHRINK) self.progressbar.set_text(_("Waiting...")) self.progressbar.set_fraction(0) self.progressbar.show() # see how many hands are in the db and adjust accordingly tcursor = self.importer.database.cursor tcursor.execute("Select count(1) from Hands") row = tcursor.fetchone() tcursor.close() self.importer.database.rollback() self.n_hands_in_db = row[0] if self.n_hands_in_db == 0: self.cb_dropindexes.set_active(2) self.cb_dropindexes.set_sensitive(False) self.lab_drop.set_sensitive(False) self.cb_drophudcache.set_active(2) self.cb_drophudcache.set_sensitive(False) self.lab_hdrop.set_sensitive(False)
import Importer Importer.loadImports('data/tiles/') from data.tiles import * class tileList: def __init__(self): self.tiles = [] self.finderList = [] key = None for key in globals().keys(): if key.split('-')[0] == 'Tile': self.tiles.append(globals()[key].setter) def getTileByType(self, world, tType): if self.finderList == []: for i in range(len(self.tiles)): self.finderList.append(self.tiles[i](world, (0, 0))) for i in range(len(self.finderList)): if self.finderList[i].tileType == tType: return self.tiles[i]
class TestImporter(unittest.TestCase): def setUp(self): self.imp = Importer() def test_startImporter(self): self.imp.startImporter() def test_getConfiguration(self): self.imp.startImporter() confFile = './fixtures/keggimporter.conf' self.imp.setConfigurationFile(confFile) config = self.imp.getConfiguration('directories', 'inserts') expectedConfig = './fixtures/inserts' self.assertEquals(config, expectedConfig) def test_openInsertFile(self): self.imp.startImporter() confFile = './fixtures/keggimporter.conf' self.imp.setConfigurationFile(confFile) result = self.imp.openInsertFile('remove_this_file') self.assertTrue(type(result) is file) result.close() def test_nextPrimaryKey(self): self.imp.startImporter() confFile = './fixtures/keggimporter.conf' self.imp.setConfigurationFile(confFile) result = self.imp.nextPrimaryKey('test') self.assertEquals(result, 1) result = self.imp.nextPrimaryKey('test') self.assertEquals(result, 2) result = self.imp.nextPrimaryKey('test') self.assertEquals(result, 3) def test_writeFile(self): self.imp.startImporter() confFile = './fixtures/keggimporter.conf' self.imp.setConfigurationFile(confFile) f = open('./fixtures/remove_this_thing', 'a') result = self.imp.writeFile(f, 'test_table', ['bla', 'ble', 'bli']) self.assertEquals(result, 1) result = self.imp.writeFile(f, 'test_table', ['bla', 'ble', 'bli']) self.assertEquals(result, 2) result = self.imp.writeFile(f, 'test_table', ['bla', 'ble', 'bli']) self.assertEquals(result, 3) def test_write_importer_files(self): self.imp.importerPathway.startImporter() confFile = './fixtures/keggimporter.conf' self.imp.importerPathway.setConfigurationFile(confFile) self.imp.importerEc.startImporter() confFile = './fixtures/keggimporter.conf' self.imp.importerEc.setConfigurationFile(confFile) self.imp.importerOrganism.startImporter() confFile = './fixtures/keggimporter.conf' self.imp.importerOrganism.setConfigurationFile(confFile) self.imp.startImporter() confFile = './fixtures/keggimporter.conf' self.imp.setConfigurationFile(confFile) self.imp.writeTaxonomies() self.imp.writePathways() self.imp.writeEcs() self.imp.writeOrganisms() self.imp.writeProteins() self.imp.writeProteinRelations() self.imp.writeOrganismTaxonomies() self.imp.writeProteinAccessions() self.imp.writeEcMaps() expectedFiles = [ 'accessionsInsert.psql', 'organismTaxonomiesInsert.psql', 'proteinMapsInsert.psql', 'ecMapsInsert.psql', 'pathwayClassesInsert.psql', 'proteinsInsert.psql', 'ecsInsert.psql', 'pathwayNamesInsert.psql', 'organismEcsInsert.psql', 'pathwaySuperClassesInsert.psql', 'taxonomiesInsert.psql', 'organismMapsInsert.psql', 'proteinAccessionsInsert.psql', 'organismsInsert.psql', 'proteinEcsInsert.psql' ] for insertFile in expectedFiles: f = open('./fixtures/inserts/' + insertFile) self.assertTrue(type(f) is file) count = 0 for line in f: count += 1 # Random number greater than 1 if count == 10: break # Random number around an expected minimum amount of records. # Empty files will raise an error (what we want to caught). if count < 5: print('EMPTY FILE: ' + insertFile) self.assertTrue(count > 5) f.close()
def __init__(self, parent, controller): tk.Frame.__init__(self, parent) self.controller = controller self.parent = parent # From here starts the code self.gimbab = importer.food('Gimbab') self.bulgogi = importer.food('Bulgogi') self.kimchi = importer.food('Kimchi_Jjigae') # In this frame the other objects will be added self.frame = tk.Frame(self, bg='white') self.frame.place(relwidth=1, relheight=1) self.image_frame = tk.Frame(self, bg='white') self.image_frame.place(relx=0.5, rely=0.2, width=400, height=450, anchor='n') # the image of the foods will be called from the get_food_image function of importer module # default image self.food_image = importer.get_food_image_path('food2') self.food_label = tk.Label(self.image_frame, image=self.food_image) self.food_label.pack() # Button to close this and go to the next page self.timer_button = tk.Button(self.frame, text='Show timer') # this keeps the bottom disabled at the beginninng. when user selects a food button will get enabled #self.timer_button.place(relx=0.5, rely=0, relwidth=0.8, relheight=0.05, anchor='n') # Button to close this and go to the next page self.button2 = tk.Button( self.frame, text='Show the recipe', command=lambda: controller.show_frame("Secondpage")) # this keeps the bottom disabled at the beginninng. when user selects a food button will get enabled self.button2["state"] = "disabled" self.button2.place(relx=0.5, rely=0.9, relwidth=0.8, relheight=0.05, anchor='n') # Its the code for dropdown menu self.dropdown_selected = tk.StringVar() self.dropdown_selected.set( "Select the desired recipe") # Default value self.food_list = (self.gimbab.foodname, self.kimchi.foodname, self.bulgogi.foodname) self.list_of_food = tk.OptionMenu(self.frame, self.dropdown_selected, *self.food_list) # StringVar().trace() : https://kite.com/python/docs/Tkinter.StringVar.trace self.dropdown_selected.trace("w", self.option_changed) self.list_of_food.place(relx=0.5, rely=0.1, relwidth=0.8, relheight=0.05, anchor='n')
log = logging.getLogger('') log.setLevel(logging.DEBUG) format = logging.Formatter( "%(asctime)s - %(name)s - %(levelname)s - %(message)s") ch = logging.StreamHandler(sys.stdout) ch.setFormatter(format) log.addHandler(ch) fh = logging.handlers.RotatingFileHandler(logFile, maxBytes=0, backupCount=0) fh.setFormatter(format) log.addHandler(fh) log.info('keggimporter: START') imp = Importer() imp.startImporter() log.info('writeTaxonomies: START') imp.writeTaxonomies() log.info('writeTaxonomies: DONE') log.info('writePathways: START') imp.writePathways() log.info('writePathways: DONE') log.info('writeEcs: START') imp.writeEcs() log.info('writeEcs: DONE')
def main(argv=None): if argv is None: argv = sys.argv[1:] Configuration.set_logfile("fpdb-log.txt") (options, argv) = Options.fpdb_options() test_all_sites = True if options.usage == True: usage() single_file_test = False if options.sitename: options.sitename = Options.site_alias(options.sitename) if options.sitename == False: usage() if options.filename: print "Testing single hand: '%s'" % options.filename single_file_test = True else: print "Only regression testing '%s' files" % (options.sitename) test_all_sites = False config = Configuration.Config(file = "HUD_config.test.xml") db = Database.Database(config) settings = {} settings.update(config.get_db_parameters()) settings.update(config.get_import_parameters()) settings.update(config.get_default_paths()) db.recreate_tables() importer = Importer.Importer(False, settings, config, None) importer.setDropIndexes("don't drop") importer.setThreads(-1) importer.setCallHud(False) importer.setFakeCacheHHC(True) AbsoluteErrors = FpdbError('Absolute Poker') BetfairErrors = FpdbError('Betfair') BetOnlineErrors = FpdbError('BetOnline') BossErrors = FpdbError('Boss') BovadaErrors = FpdbError('Bovada') CakeErrors = FpdbError('Cake') EnetErrors = FpdbError('Enet') EntractionErrors = FpdbError('Entraction') EverleafErrors = FpdbError('Everleaf Poker') EverestErrors = FpdbError('Everest Poker') FTPErrors = FpdbError('Full Tilt Poker') iPokerErrors = FpdbError('iPoker') MergeErrors = FpdbError('Merge') MicrogamingErrors = FpdbError('Microgaming') OnGameErrors = FpdbError('OnGame') PacificPokerErrors= FpdbError('PacificPoker') PartyPokerErrors = FpdbError('Party Poker') PokerStarsErrors = FpdbError('PokerStars') PKRErrors = FpdbError('PKR') PTErrors = FpdbError('PokerTracker') WinamaxErrors = FpdbError('Winamax') ErrorsList = [ AbsoluteErrors, BetfairErrors, BetOnlineErrors, BossErrors, CakeErrors, EntractionErrors, EverleafErrors, EverestErrors, FTPErrors, iPokerErrors, MergeErrors, MicrogamingErrors, OnGameErrors, PacificPokerErrors, PartyPokerErrors, PokerStarsErrors, PKRErrors, PTErrors, WinamaxErrors, BovadaErrors, EnetErrors, ] sites = { 'Absolute' : False, 'Betfair' : False, 'BetOnline': False, 'Boss' : False, 'Bovada' : False, 'Cake' : False, 'Enet' : False, 'Entraction' : False, 'Everleaf' : False, 'Everest' : False, 'Full Tilt Poker' : False, 'iPoker' : False, 'Merge' : False, 'Microgaming': False, 'OnGame' : False, 'Pkr' : False, 'PacificPoker' : False, 'PartyPoker' : False, 'PokerStars' : False, 'PokerTracker' : False, 'Winamax' : False, } if test_all_sites == True: for s in sites: sites[s] = True else: sites[options.sitename] = True if sites['PacificPoker'] == True and not single_file_test: walk_testfiles("regression-test-files/cash/PacificPoker/", compare, importer, PacificPokerErrors, "PacificPoker") walk_testfiles("regression-test-files/tour/PacificPoker/", compare, importer, PacificPokerErrors, "PacificPoker") walk_testfiles("regression-test-files/summaries/PacificPoker/", compare, importer, PacificPokerErrors, "PacificPoker") elif sites['PacificPoker'] == True and single_file_test: walk_testfiles(options.filename, compare, importer, PacificPokerErrors, "PacificPoker") if sites['PokerStars'] == True and not single_file_test: walk_testfiles("regression-test-files/cash/Stars/", compare, importer, PokerStarsErrors, "PokerStars") walk_testfiles("regression-test-files/tour/Stars/", compare, importer, PokerStarsErrors, "PokerStars") walk_testfiles("regression-test-files/summaries/Stars/", compare, importer, PokerStarsErrors, "PokerStars") elif sites['PokerStars'] == True and single_file_test: walk_testfiles(options.filename, compare, importer, PokerStarsErrors, "PokerStars") if sites['Full Tilt Poker'] == True and not single_file_test: walk_testfiles("regression-test-files/cash/FTP/", compare, importer, FTPErrors, "Full Tilt Poker") walk_testfiles("regression-test-files/tour/FTP/", compare, importer, FTPErrors, "Full Tilt Poker") walk_testfiles("regression-test-files/summaries/FTP/", compare, importer, FTPErrors, "Full Tilt Poker") elif sites['Full Tilt Poker'] == True and single_file_test: walk_testfiles(options.filename, compare, importer, FTPErrors, "Full Tilt Poker") if sites['PartyPoker'] == True and not single_file_test: walk_testfiles("regression-test-files/cash/PartyPoker/", compare, importer, PartyPokerErrors, "PartyPoker") walk_testfiles("regression-test-files/tour/PartyPoker/", compare, importer, PartyPokerErrors, "PartyPoker") elif sites['PartyPoker'] == True and single_file_test: walk_testfiles(options.filename, compare, importer, PartyPokerErrors, "PartyPoker") if sites['Betfair'] == True and not single_file_test: walk_testfiles("regression-test-files/cash/Betfair/", compare, importer, BetfairErrors, "Betfair") elif sites['Betfair'] == True and single_file_test: walk_testfiles(options.filename, compare, importer, BetfairErrors, "Betfair") if sites['OnGame'] == True and not single_file_test: walk_testfiles("regression-test-files/cash/OnGame/", compare, importer, OnGameErrors, "OnGame") walk_testfiles("regression-test-files/tour/OnGame/", compare, importer, OnGameErrors, "OnGame") elif sites['OnGame'] == True and single_file_test: walk_testfiles(options.filename, compare, importer, OnGameErrors, "OnGame") if sites['Absolute'] == True and not single_file_test: walk_testfiles("regression-test-files/cash/Absolute/", compare, importer, AbsoluteErrors, "Absolute") walk_testfiles("regression-test-files/tour/Absolute/", compare, importer, AbsoluteErrors, "Absolute") elif sites['Absolute'] == True and single_file_test: walk_testfiles(options.filename, compare, importer, AbsoluteErrors, "Absolute") if sites['Everleaf'] == True and not single_file_test: walk_testfiles("regression-test-files/cash/Everleaf/", compare, importer, EverleafErrors, "Everleaf") walk_testfiles("regression-test-files/tour/Everleaf/", compare, importer, EverleafErrors, "Everleaf") elif sites['Everleaf'] == True and single_file_test: walk_testfiles(options.filename, compare, importer, EverleafErrors, "Everleaf") if sites['Everest'] == True and not single_file_test: walk_testfiles("regression-test-files/cash/Everest/", compare, importer, EverestErrors, "Everest") walk_testfiles("regression-test-files/tour/Everest/", compare, importer, EverestErrors, "Everest") elif sites['Everest'] == True and single_file_test: walk_testfiles(options.filename, compare, importer, EverestErrors, "Everest") if sites['Merge'] == True and not single_file_test: walk_testfiles("regression-test-files/cash/Merge/", compare, importer, MergeErrors, "Merge") walk_testfiles("regression-test-files/tour/Merge/", compare, importer, MergeErrors, "Merge") elif sites['Merge'] == True and single_file_test: walk_testfiles(options.filename, compare, importer, MergeErrors, "Merge") if sites['Pkr'] == True and not single_file_test: walk_testfiles("regression-test-files/cash/PKR/", compare, importer, PKRErrors, "PKR") walk_testfiles("regression-test-files/tour/PKR/", compare, importer, PKRErrors, "PKR") elif sites['Pkr'] == True and single_file_test: walk_testfiles(options.filename, compare, importer, PKRErrors, "PKR") if sites['iPoker'] == True and not single_file_test: walk_testfiles("regression-test-files/cash/iPoker/", compare, importer, iPokerErrors, "iPoker") walk_testfiles("regression-test-files/tour/iPoker/", compare, importer, iPokerErrors, "iPoker") elif sites['iPoker'] == True and single_file_test: walk_testfiles(options.filename, compare, importer, iPokerErrors, "iPoker") if sites['Boss'] == True and not single_file_test: walk_testfiles("regression-test-files/cash/Boss/", compare, importer, BossErrors, "Boss") walk_testfiles("regression-test-files/tour/Boss/", compare, importer, BossErrors, "Boss") elif sites['Boss'] == True and single_file_test: walk_testfiles(options.filename, compare, importer, BossErrors, "Boss") if sites['Entraction'] == True and not single_file_test: walk_testfiles("regression-test-files/cash/Entraction/", compare, importer, EntractionErrors, "Entraction") walk_testfiles("regression-test-files/tour/Entraction/", compare, importer, EntractionErrors, "Entraction") elif sites['Entraction'] == True and single_file_test: walk_testfiles(options.filename, compare, importer, EntractionErrors, "Entraction") if sites['BetOnline'] == True and not single_file_test: walk_testfiles("regression-test-files/cash/BetOnline/", compare, importer, BetOnlineErrors, "BetOnline") walk_testfiles("regression-test-files/tour/BetOnline/", compare, importer, BetOnlineErrors, "BetOnline") elif sites['BetOnline'] == True and single_file_test: walk_testfiles(options.filename, compare, importer, BetOnlineErrors, "BetOnline") if sites['Microgaming'] == True and not single_file_test: walk_testfiles("regression-test-files/cash/Microgaming/", compare, importer, MicrogamingErrors, "Microgaming") walk_testfiles("regression-test-files/tour/Microgaming/", compare, importer, MicrogamingErrors, "Microgaming") elif sites['Microgaming'] == True and single_file_test: walk_testfiles(options.filename, compare, importer, MicrogamingErrors, "Microgaming") if sites['Cake'] == True and not single_file_test: walk_testfiles("regression-test-files/cash/Cake/", compare, importer, CakeErrors, "Cake") walk_testfiles("regression-test-files/tour/Cake/", compare, importer, CakeErrors, "Cake") elif sites['Cake'] == True and single_file_test: walk_testfiles(options.filename, compare, importer, CakeErrors, "Cake") if sites['PokerTracker'] == True and not single_file_test: walk_testfiles("regression-test-files/cash/PokerTracker/", compare, importer, PTErrors, "PokerTracker") walk_testfiles("regression-test-files/tour/PokerTracker/", compare, importer, PTErrors, "PokerTracker") elif sites['PokerTracker'] == True and single_file_test: walk_testfiles(options.filename, compare, importer, PTErrors, "PokerTracker") if sites['Winamax'] == True and not single_file_test: walk_testfiles("regression-test-files/cash/Winamax/", compare, importer, WinamaxErrors, "Winamax") walk_testfiles("regression-test-files/tour/Winamax/", compare, importer, WinamaxErrors, "Winamax") elif sites['Winamax'] == True and single_file_test: walk_testfiles(options.filename, compare, importer, WinamaxErrors, "Winamax") if sites['Bovada'] == True and not single_file_test: walk_testfiles("regression-test-files/cash/Bovada/", compare, importer, BovadaErrors, "Bovada") walk_testfiles("regression-test-files/tour/Bovada/", compare, importer, BovadaErrors, "Bovada") elif sites['Bovada'] == True and single_file_test: walk_testfiles(options.filename, compare, importer, BovadaErrors, "Bovada") if sites['Enet'] == True and not single_file_test: walk_testfiles("regression-test-files/cash/Enet/", compare, importer, EnetErrors, "Enet") elif sites['Enet'] == True and single_file_test: walk_testfiles(options.filename, compare, importer, EnetErrors, "Enet") totalerrors = 0 for i, site in enumerate(ErrorsList): totalerrors += ErrorsList[i].errorcount for i, site in enumerate(ErrorsList): ErrorsList[i].print_histogram() # Merge the dicts of stats from the various error objects statdict = {} for i, site in enumerate(ErrorsList): tmp = ErrorsList[i].statcount for stat in tmp: if stat in statdict: statdict[stat] += tmp[stat] else: statdict[stat] = tmp[stat] print "\n" print "---------------------" print "Errors by stat:" print "---------------------" #for stat in statdict: # print "(%3d) : %s" %(statdict[stat], stat) sortedstats = sorted([(value,key) for (key,value) in statdict.items()]) for num, stat in sortedstats: print "(%3d) : %s" %(num, stat) print "---------------------" print "Total Errors: %d" % totalerrors print "---------------------" print "-------- Parse Error List --------" for i, site in enumerate(ErrorsList): ErrorsList[i].print_parse_list()
import copy, Importer Importer.loadImports('data/items/') from data.items import * class itemList: def __init__(self, itemPictures): self.itemPictures = itemPictures self.items = [] self.finderList = [] key = None for key in globals().keys(): if key.split('-')[0] == 'Item': self.items.append(globals()[key].setter) def getItems(self): return self.items def getItemByID(self, ID): if self.finderList == []: for i in range(len(self.items)): self.finderList.append(self.items[i]()) for i in range(len(self.finderList)): if self.finderList[i].ID == ID: return self.items[i] def getPictures(self): return self.itemPictures
import Importer Importer.loadImports('data/tiles/') from data.tiles import * class tileList: def __init__(self): self.tiles = [] self.finderList = [] key = None for key in globals().keys(): if key.split('-')[0] == 'Tile': self.tiles.append(globals()[key].setter) def getTileByType(self,world,tType): if self.finderList == []: for i in range(len(self.tiles)): self.finderList.append(self.tiles[i](world, (0,0))) for i in range(len(self.finderList)): if self.finderList[i].tileType == tType: return self.tiles[i]
def main(input_path, output_path, days, max_missing_days): weekdays = (5 * days) / 7 output_dir = os.path.dirname(os.path.abspath(output_path)) if not os.path.exists(output_dir): os.makedirs(output_dir) kill_path = output_dir + '/kill' killfile_monitor = KillFileMonitor(kill_path, 1) output_path_temp = output_path + '.incomplete' try: with open(output_path_temp, 'w', encoding='utf8') as out_file: out_file.write('id;instrument;decision;confirmed;time;') for day in range(-days + 1, 1): out_file.write(str(day)) if day < 0: out_file.write(';') progress = FileItemProgress('flatten: ', 1, None, None) known_ids = set() for file_path in sorted(glob.iglob(input_path), reverse=True): killfile_monitor.maybe_check_killfile() with open(file_path, 'r', encoding='utf8') as in_file: progress.set_file(file_path, in_file) def flatten_snapshot(snapshot): killfile_monitor.maybe_check_killfile() id = snapshot.DecisionId if id in known_ids: return if snapshot.Decision is not None and snapshot.snapshotrates is not None: known_ids.add(id) snapshot_date = datetime.datetime.strptime( snapshot.Time, '%Y-%m-%d %H:%M:%S').date() first_date = snapshot_date - datetime.timedelta( days - 1) rates = list( filter( lambda r: r.Close is not None and r.Close > 0, snapshot.snapshotrates)) rates = list( map( lambda r: Rate( datetime.datetime.strptime( r.Time, '%Y-%m-%d %H:%M:%S').date( ), r.Close), rates)) if len(rates) > 0: previous_rate = lastOrDefault( (r for r in rates if r.Date < first_date), rates[0]) remaining_rates = list( itertools.dropwhile( lambda r: r.Date < first_date, rates)) if len(remaining_rates) >= (weekdays - max_missing_days): out_file.write('\n') out_file.write(str(id)) out_file.write(';') out_file.write(str(snapshot.instrument.ID)) out_file.write(';') out_file.write(snapshot.Decision) out_file.write(';') out_file.write(str(snapshot.Confirmed)) out_file.write(';') out_file.write( snapshot_date.strftime('%Y%m%d')) out_file.write(';') snapshot_days = (snapshot_date - first_date).days + 1 for day in range(snapshot_days): date = first_date + datetime.timedelta( days=day) remaining_rates = list( itertools.dropwhile( lambda r: r.Date < date, remaining_rates)) rate = previous_rate if len(remaining_rates) > 0: firstRate = remaining_rates[0] if firstRate.Date == date: rate = firstRate # else: #encountered future rate. use previous rate. # else: #no remaining rates. use previous rate. value = '%.2f' % (rate.Close) out_file.write(value) if day < snapshot_days - 1: out_file.write(';') previous_rate = rate else: print_flush( "%s has insufficient rates: %d < %d - %d" % (snapshot.instrument.InstrumentName, len(remaining_rates), weekdays, max_missing_days)) progress.add_item() progress.maybe_print() importer = Importer() importer.import_stream(in_file, flatten_snapshot) if os.path.exists(output_path): os.remove(output_path) os.rename(output_path_temp, output_path) except KilledException: killfile_monitor.delete_killfile() if os.path.exists(output_path_temp): os.remove(output_path_temp) print_flush('Killed.')
def main(): """Starts the tests.""" logging.basicConfig(filename=ERROR_LOG, filemode='w', level=logging.DEBUG, format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') successes = [] failures = [] # Parse the command line arguments. parser = argparse.ArgumentParser() parser.add_argument("--dir", type=str, action="store", default=os.path.dirname(os.path.realpath(__file__)), help="Directory of files to be processed", required=True) args = parser.parse_args() store = TestActivityWriter() importer = Importer.Importer(store) test_dir = os.path.abspath(os.path.join('.', args.dir)) # Process each file in the specified directory as well as its subdirectories. total_time = 0 num_files_processed = 0 for subdir, _, files in os.walk(test_dir): title_str = "Processing all files in " + test_dir + ":" print(title_str + "\n") for current_file in files: # My test file repo has a description file that we should skip. if current_file == "description.csv": continue full_path = os.path.join(subdir, current_file) _, temp_file_ext = os.path.splitext(full_path) if temp_file_ext in ['.gpx', '.tcx', '.csv']: title_str = "Processing: " + full_path print("=" * len(title_str)) print(title_str) print("=" * len(title_str)) start_time = time.time() success, device_id, activity_id = importer.import_file( "test user", "", full_path, current_file, temp_file_ext) if success: elapsed_time = time.time() - start_time total_time = total_time + elapsed_time num_files_processed = num_files_processed + 1 print("Elapsed Processing Time: " + str(elapsed_time) + " seconds") print("Success!\n") successes.append(current_file) else: print("Failure!\n") failures.append(current_file) # Print the summary data. print_records(store, Keys.TYPE_RUNNING_KEY) print_records(store, Keys.TYPE_CYCLING_KEY) print_records(store, Keys.TYPE_OPEN_WATER_SWIMMING_KEY) print_records(store, Keys.TYPE_POOL_SWIMMING_KEY) # Print the maximum heart rate and heart rate zone calculators. max_hr = store.summarizer.hr_calc.estimate_max_hr(45) print("Estimated Maximum Heart Rate: {:.2f} bpm\n".format(max_hr)) # Print the success and failure summary. title_str = "Summary:" print(title_str) print("=" * len(title_str)) print("Num success: " + str(len(successes))) print("Num failures: " + str(len(failures))) for failure in failures: print("- " + failure) # Print the time summary. if num_files_processed > 0: print("Average time per sample: " + str(total_time / num_files_processed) + " seconds\n") else: print("No files processed.\n")
def __init__(self): self.objects = {} self.importer = Importer()
total_iterations *= 1 print(f"Total iterations: {total_iterations}") step = 0 material_number = 0 state = {"negative": False, "positive": False} margin_list = [] mass_list = [] margin_details = [] input_details = [] first_iteration = True print(f"-- Importing materials --") material_list = Importer.import_all_materials("material_sheet", ("metal", "none")) print(material_list[0].type, material_list[1].type, material_list[2].type, material_list[3].type, material_list[4].type, material_list[5].type, material_list[6].type) material_properties_attachment = material_list[6] material_properties_fastener = material_list[6] material_properties_vehicle = material_list[6] print( f"Material: {material_properties_attachment.name} {material_properties_attachment.type}" ) material_details = [ material_properties_attachment.name, material_properties_attachment.type ] material_number += 1
#!/usr/bin/env python import Splitter import Tagger import Scorer import Importer import yaml from pprint import pprint if __name__ == "__main__": rss = Importer.RSSImporter( 'https://news.google.com/news/feeds?q=apple&output=rss') input_text = rss.parse() s = Splitter.Splitter() tagger = Tagger.DictionaryTagger([ 'dicts/positive.yml', 'dicts/negative.yml', 'dicts/inc.yml', 'dicts/dec.yml', 'dicts/inv.yml' ]) scorer = Scorer.Scorer() total = 0 for summary in input_text: split = s.split(summary) tagged = tagger.tag(split) score = scorer.score(tagged) print "%s -> %d" % (summary, score) total += score
def import_activity(import_str, internal_task_id): local_file_name = "" try: import_obj = json.loads(import_str) username = import_obj['username'] user_id = import_obj['user_id'] uploaded_file_data = import_obj['uploaded_file_data'] uploaded_file_name = import_obj['uploaded_file_name'] desired_activity_id = import_obj['desired_activity_id'] data_mgr = DataMgr.DataMgr(None, "", None, None, None) importer = Importer.Importer(data_mgr) # Generate a random name for the local file. print("Generating local file name...") root_dir = os.path.dirname(os.path.abspath(__file__)) tempfile_dir = os.path.join(root_dir, 'tempfile') if not os.path.exists(tempfile_dir): os.makedirs(tempfile_dir) upload_path = os.path.normpath(tempfile_dir) uploaded_file_name, uploaded_file_ext = os.path.splitext( uploaded_file_name) local_file_name = os.path.join(upload_path, str(uuid.uuid4())) local_file_name = local_file_name + uploaded_file_ext # Decode and write the file. print("Writing the data to a local file...") with open(local_file_name, 'wb') as local_file: # Data to import is expected to be Base 64 encoded. This is because, at this point, we don't distinguish between # text and binary files. print("Base64 decoding...") uploaded_file_data = uploaded_file_data.replace( " ", "+" ) # Some JS base64 encoders replace plus with space, so we need to undo that. decoded_file_data = base64.b64decode(uploaded_file_data) # print("zlib decompressing...") # decoded_file_data = zlib.decompress(decoded_file_data) print("Writing...") local_file.write(decoded_file_data) # Update the status of the analysis in the database. print("Updating status...") data_mgr.update_deferred_task(user_id, internal_task_id, None, Keys.TASK_STATUS_STARTED) # Import the file into the database. print("Importing the data to the database...") success, _, activity_id = importer.import_activity_from_file( username, user_id, local_file_name, uploaded_file_name, uploaded_file_ext, desired_activity_id) # The import was successful, do more stuff. if success: # Save the file to the database. print("Saving the file to the database...") data_mgr.create_uploaded_file(activity_id, decoded_file_data) # Update the status of the analysis in the database. print("Updating status...") data_mgr.update_deferred_task(user_id, internal_task_id, activity_id, Keys.TASK_STATUS_FINISHED) # Schedule the activity for analysis. print("Importing was successful, performing analysis...") data_mgr.analyze_activity_by_id(activity_id, user_id) # The import failed. else: # Update the status of the analysis in the database. print("Import was not successful.") data_mgr.update_deferred_task(user_id, internal_task_id, activity_id, Keys.TASK_STATUS_ERROR) except: log_error("Exception when importing activity data: " + str(import_str)) log_error(traceback.format_exc()) log_error(sys.exc_info()[0]) finally: # Remove the local file. if len(local_file_name) > 0: print("Removing local file...") os.remove(local_file_name)
def main(argv=None): if argv is None: argv = sys.argv run_importer = True run_formatter = True run_analysis = True try: try: opts, args = getopt.getopt(argv[1:], "h", ["help", "skip-importer", "skip-formatter", "skip-analysis"]) print "got args" except getopt.error, msg: raise Usage(msg) for option, data in opts: if('-h' == option or '--help' == option): print "LSCE test script. Usage: \"python testscript.py [--skip-importer] [--skip-formatter] [--skip-analysis]" +\ " mat_source hdf5_dest" +\ "\"\n\nSupply the following arguments to run pipeline:\n\n\tmat_source: " +\ "The path to the raw .mat files to be imported.\n\thdf5_dest: the name to save hdf5 output file under" +\ "\n\nAvailable modes:\n\t--skip-importer: skip importation step. Formatter wil still run using" +\ " mat_source as src directory." +\ "\n\t--skip-formatter: skip formatting step. Importer will use mat_source as usual. \n\t\t\t Analysis will" +\ " use hdf5_dest if it exists." + \ "\n\t--skip-analysis: skip computation of analysis data. Formatter will still output to hdf5_dest. " return if('--skip-importer' == option): run_importer = False if('--skip-formatter' == option): run_formatter = False if('--skip-analysis' == option): run_analysis = False if(len(args) < 2): raise Usage("Insufficient arguments supplied.") else: print args.__repr__() print "Welcome to LSCE test script.\nThis script will perform a " + \ "complete iteration of our pipeline, starting with the data importer." if(run_importer): print "Importing data from directory "+args[0] Importer.loadFromMat(args[0]) else: print "Skipped importing data." if(run_formatter): print "Formatting data as hdf5 in "+args[1]+".hdf5" DataFormatter.formatData(args[0], args[1]) else: print "Skipped formatting data." os.system("PAUSE") testing = None raw_data = None if(run_analysis): dtool = DataAnalysis.data_analysis() dtool.load_hdf5(args[1], dataset_name="Electrode_12_master", group_name="raw_data") dtool.sampling_rate = 1000 testing = dtool.high_demo_filter(20) raw_data = dtool.f["raw_data"]["Electrode_12_master"] else: print "Skipped data analysis.\nPlaceholder groups " + \ "\"/data_analysis/demo_filter_results\" and \"/raw_data/Electrode_12_master\" will be used." hdfile = h5py.File(args[1]+".hdf5", "r+") if("data_analysis" not in hdfile or "demo_filter_results" not in hdfile["data_analysis"]): print "Skipping graphs..." return testing = hdfile["data_analysis"]["demo_filter_results"] raw_data = hdfile["raw_data"]["Electrode_12_master"] os.system("PAUSE") plt.subplot(2, 1, 1) plt.plot(testing) plt.subplot(2, 1, 2) plt.plot(raw_data) plt.show() if(run_analysis): dtool.close()
import Importer Importer.loadImports('data/blueprints/') from data.blueprints import * class blueprint: def __init__(self,ID,width,height,tiles): self.ID = ID self.tiles = tiles self.width = width self.height = height def getRel(self,colRow): toRet = [] for t in range(len(self.tiles)): toRet.append((colRow[1]-self.tiles[t][0],self.tiles[t][1] + colRow[0],self.tiles[t][2],self.tiles[t][3])) return toRet def getRequiredResources(self): req = [] for t in range(len(self.tiles)): found = False for i in range(len(req)): if req[i][0] == self.tiles[t][3]: found = True if found == True: for l in range(len(req)): if req[l][0] == self.tiles[t][3]:
def run_unit_tests(): """Entry point for the unit tests.""" successes = [] failures = [] store = TestActivityWriter() importer = Importer.Importer(store) test_dir = os.path.abspath(os.path.join('.', args.dir)) # Process each file in the specified directory as well as its subdirectories. total_time = 0 num_files_processed = 0 for subdir, _, files in os.walk(test_dir): title_str = "Processing all files in " + test_dir + ":" print(title_str + "\n") for current_file in files: # My test file repo has a description file that we should skip. if current_file == "description.csv": continue full_path = os.path.join(subdir, current_file) _, temp_file_ext = os.path.splitext(full_path) if temp_file_ext in ['.gpx', '.tcx', '.csv']: title_str = "Processing: " + full_path print("=" * len(title_str)) print(title_str) print("=" * len(title_str)) start_time = time.time() success, device_id, activity_id = importer.import_activity_from_file( "test user", "", full_path, current_file, temp_file_ext) if success: elapsed_time = time.time() - start_time total_time = total_time + elapsed_time num_files_processed = num_files_processed + 1 print(f"Elapsed Processing Time: {elapsed_time} seconds") print("Success!\n") successes.append(current_file) else: print("Failure!\n") failures.append(current_file) # Print the summary data. print_records(store, Keys.TYPE_RUNNING_KEY) print_records(store, Keys.TYPE_CYCLING_KEY) print_records(store, Keys.TYPE_OPEN_WATER_SWIMMING_KEY) print_records(store, Keys.TYPE_POOL_SWIMMING_KEY) # Print the maximum heart rate and heart rate zone calculators. max_hr = store.summarizer.hr_calc.estimate_max_hr(45) print("Estimated Maximum Heart Rate: {:.2f} bpm\n".format(max_hr)) # Print the success and failure summary. title_str = "Summary:" print(title_str) print("=" * len(title_str)) print("Num success: " + str(len(successes))) print("Num failures: " + str(len(failures))) for failure in failures: print("- " + failure) # Print the time summary. if num_files_processed > 0: print("Average time per sample: " + str(total_time / num_files_processed) + " seconds\n") else: print("No files processed.\n") return len(failures) == 0
def main(argv=None): if argv is None: argv = sys.argv run_importer = True run_formatter = True run_analysis = True try: try: opts, args = getopt.getopt( argv[1:], "h", ["help", "skip-importer", "skip-formatter", "skip-analysis"]) print "got args" except getopt.error, msg: raise Usage(msg) for option, data in opts: if ('-h' == option or '--help' == option): print "LSCE test script. Usage: \"python testscript.py [--skip-importer] [--skip-formatter] [--skip-analysis]" +\ " mat_source hdf5_dest" +\ "\"\n\nSupply the following arguments to run pipeline:\n\n\tmat_source: " +\ "The path to the raw .mat files to be imported.\n\thdf5_dest: the name to save hdf5 output file under" +\ "\n\nAvailable modes:\n\t--skip-importer: skip importation step. Formatter wil still run using" +\ " mat_source as src directory." +\ "\n\t--skip-formatter: skip formatting step. Importer will use mat_source as usual. \n\t\t\t Analysis will" +\ " use hdf5_dest if it exists." + \ "\n\t--skip-analysis: skip computation of analysis data. Formatter will still output to hdf5_dest. " return if ('--skip-importer' == option): run_importer = False if ('--skip-formatter' == option): run_formatter = False if ('--skip-analysis' == option): run_analysis = False if (len(args) < 2): raise Usage("Insufficient arguments supplied.") else: print args.__repr__() print "Welcome to LSCE test script.\nThis script will perform a " + \ "complete iteration of our pipeline, starting with the data importer." if (run_importer): print "Importing data from directory " + args[0] Importer.loadFromMat(args[0]) else: print "Skipped importing data." if (run_formatter): print "Formatting data as hdf5 in " + args[1] + ".hdf5" DataFormatter.formatData(args[0], args[1]) else: print "Skipped formatting data." os.system("PAUSE") testing = None raw_data = None if (run_analysis): dtool = DataAnalysis.data_analysis() dtool.load_hdf5(args[1], dataset_name="Electrode_12_master", group_name="raw_data") dtool.sampling_rate = 1000 testing = dtool.high_demo_filter(20) raw_data = dtool.f["raw_data"]["Electrode_12_master"] else: print "Skipped data analysis.\nPlaceholder groups " + \ "\"/data_analysis/demo_filter_results\" and \"/raw_data/Electrode_12_master\" will be used." hdfile = h5py.File(args[1] + ".hdf5", "r+") if ("data_analysis" not in hdfile or "demo_filter_results" not in hdfile["data_analysis"]): print "Skipping graphs..." return testing = hdfile["data_analysis"]["demo_filter_results"] raw_data = hdfile["raw_data"]["Electrode_12_master"] os.system("PAUSE") plt.subplot(2, 1, 1) plt.plot(testing) plt.subplot(2, 1, 2) plt.plot(raw_data) plt.show() if (run_analysis): dtool.close()