def _connect_to_sqlite(self, db): sqlite_db_path = db.sqlite_file if not os.path.exists(sqlite_db_path): raise Exception( "Sqlite database not found at '%s', indicating that either the SingleM database was built with an out-dated SingleM version, or that the database is corrupt. Please generate a new database with the current version of SingleM." ) logging.debug("Connecting to %s" % sqlite_db_path) dbm = DatabaseManager( {'sqlite3': { 'driver': 'sqlite', 'database': sqlite_db_path }}) Model.set_connection_resolver(dbm) try: len(dbm.table('otus').limit(1).get()) except Exception as e: logging.error( "Failure to extract any data from the otus table of the SQ Lite DB indicates this SingleM DB is either too old or is corrupt." ) raise (e) try: len(dbm.table('clusters').limit(1).get()) except QueryException: logging.error( "Failure to extract any data from the 'clusters' table indicates this SingleM DB is out-dated, and cannot be used with query implemented in this version of SingleM" ) sys.exit(1) return dbm
def load_data(): try: db = DatabaseManager(app.config.get('DB_CONFIG')).connection() if not db.table('greetings').get(['message']).first(): db.table('greetings').insert({'message': 'Hello world!'}) except Exception as e: app.logger.error('Unable to connect to db: %s' % e)
class WriterWorker(Thread): def __init__(self, task, worker_control): Thread.__init__(self) self.task = task self.worker_control = worker_control self.db = DatabaseManager(DATABASES) def update_task_status(self, params): print("- Updating processing_status for task id %s with processing_status of %s into the database" % (params['id'], params['processing_status'])) self.db.table("tasks").where("id", params['id']).update({"ipfs_hash": params["ipfs_hash"], "processing_status": params['processing_status']}) print("-- Done updating the task-id %s into the database" % params['id']) def run(self): print("Writer worker online for task-id: %s" % self.task["id"]) try: self.update_task_status(self.task) except Exception as err: print(err) self.worker_control.release()
def get_or_create_file(self, db: orator.DatabaseManager, filepath: Union[str, pathlib.Path], metadata: Union[str, dict, None] = None) -> dict: # enforce types checks.check_types(db, orator.DatabaseManager) checks.check_types(filepath, [str, pathlib.Path]) checks.check_types(metadata, [str, dict, type(None)]) # convert types filepath = pathlib.Path(filepath) if isinstance(metadata, dict): metadata = str(dict) # check file exists checks.check_file_exists(filepath) # check exists md5 = tools.get_file_hash(filepath) sha256 = tools.get_file_hash(filepath, hashlib.sha256) file_info = self.get_file(db=db, md5=md5) # return if found if file_info is not None: return file_info name = "fms_" + md5 # create if not with tools.suppress_prints(): self._build_file_as_package(filepath, name) # import string read_pkg = importlib.import_module(name="quilt.data." + self.storage_user + "." + name) # file info dict file_info = { "FileId": str(uuid.uuid4()), "OriginalFilepath": str(filepath), "FileType": filepath.suffix[1:], "ReadPath": read_pkg.load(), "MD5": md5, "SHA256": sha256, "Metadata": metadata, "Created": datetime.utcnow() } # insert db.table("File").insert(file_info) return file_info
def Alquilar(self): self.__log.info("Alquilar un libro") estadolibro = EstadoLibro() print(f"\t ID\t Descripcion") for obj in estadolibro.all(): print(f"\t {obj.id}\t {obj.descripcion}") if obj.descripcion == 1: libroporalquilar = Prestamo() cliente = User() print(f"\t ID\t Nombre\t CORREO\t DOCUMENTO\t ESTADO") for obj in cliente.all(): print(f"\t {obj.id}\t {obj.nombre}\t {obj.correo}\t {obj.documento}\t {obj.estado_user_id}") print("Escriba el id del cliente") cliente = input() libro = Libro() print(f"\t ID\t Nombre\t ISBN\t Autor\t Editorial\t Estado") for obj in libro.all(): print(f"\t {obj.id}\t {obj.nombre}\t {obj.isbn}\t {obj.autors_id}\t {obj.editorial_id}\t {obj.estado_libro_id}") print("Escriba el id del libro") libro = input() DatabaseManager.table('libro').where('nombre', f'{obj.nombre}').update({'obj.estado_libro_id': 3}) biblioteca = Biblioteca() print(f"\t ID\t Nombre\t DIRECCION") for obj in biblioteca.all(): print(f"\t {obj.id}\t {obj.nombre}\t {obj.direccion}") print("Escriba el id de la biblioteca") biblioteca = input() libroporalquilar.users_id = cliente libroporalquilar.libros_id = libro libroporalquilar.bibliotecas_id = biblioteca libroporalquilar.save() objMenus = init.Inicio() objMenus.MenuInicioMenu() elif obj.descripcion == 2: print("Lo sentimos el libro que busca esta reservado") objMenus = init.Inicio() objMenus.MenuInicioMenu() elif obj.descripcion == 2: print("Lo sentimos el libro que busca no se encuentra disponible") objMenus = init.Inicio() objMenus.MenuInicioMenu()
def RegistroEditorial(self): menuRegisEdit = True __log = log("Ingresando al Registro de las editoriales") self.__log.info("Ingresando al Registro de las editoriales") opcionesRegisEditorial = { "Registrar Editorial": 1, "Listar Editoriales": 2, "Eliminar Editoriales": 3 } MenuRegisEditorial = Menu("Menu Modulo Libros", opcionesRegisEditorial) regisEditoriales = MenuRegisEditorial.mostrarMenu() menuRegisEdit = True while menuRegisEdit: if (regisEditoriales == 1): nuevaEditorial = Editorial() nombreEditorial = input("escriba el nombre de la editorial \n") nuevaEditorial.nombre = nombreEditorial nuevaEditorial.save() menuRegisEdit = False elif (regisEditoriales == 2): editorial = Editorial() print(f"\t ID\t NOMBRE") for obj in editorial.all(): print(f"\t {obj.id}\t {obj.nombre}") input("Regresar???") program.utils.Menu("Menu Modulo Editoriales", regisEditoriales) elif (regisEditoriales == 3): editorial = Editorial() print(f"\t ID\t NOMBRE") for obj in editorial.all(): print(f"\t {obj.id}\t {obj.nombre}") print( "Escriba el id de la editorial que se desea eliminar: \n") id = input() DatabaseManager.table('editorial').get() DatabaseManager.table('editorial').where('id', '=', f'{id}').delete() elif (regisEditoriales == 9): __log = log("Saliendo") break
class Loader(Process): def __init__(self): super(Loader, self).__init__() self.db = DatabaseManager(DATABASES) def run(self): print("*** Loader online") images = get_all_images() ##get all images in list and put that images in the database and set to new PENDING if images: for image in images: print("Inserting image file-path %s into the database" %image) self.db.table("tasks").insert({'image_path': image})
class MainOperate: def __init__(self): self.db = DatabaseManager(config=CONFIG).connection() def one_mysql_link(self): """ 根据offset limit 查询数据 :return: """ result = self.db.table("corpus").offset(2).limit(5).get() # 对于多条件查询 db_data = self.db.table("corpus").where('type', 'all').where_between( 'age', [15, 25]).get() for num, data in enumerate(result): # 根据字段名获取每一条信息 logging.info(data.get("url")) return result def two_add_column(self): """ 给已存在的数据表增加一列数据 :return: """ new_data = { "title": "测试标题", "url": "https://www.baidu.com", "content": "测试正文", "published": "2019-11-02 12:16:01", } result = self.db.table("corpus").insert(new_data) logging.info(result) def three_update_data(self): """ 更新数据库操作 :return: """ corpus = self.db.table("corpus").where('id', '1').update( {'published': '2019-12-03 09:25:01'}) logging.info(corpus) def four_delete_data(self): """ 根据id来删除一行数据 :return: """ delete_data = self.db.table("corpus").delete('1899') logging.info(delete_data)
def reconstruct(db: orator.DatabaseManager, ds_info: "DatasetInfo", fms: FMSInterface) -> pd.DataFrame: # get all iota that match data = [ dict(r) for r in db.table("Iota"). join("IotaGroup", "IotaGroup.IotaId", "=", "Iota.IotaId").join( "GroupDataset", "GroupDataset.GroupId", "=", "IotaGroup.GroupId" ).join("Dataset", "GroupDataset.DatasetId", "=", "Dataset.DatasetId" ).where("Dataset.DatasetId", "=", ds_info.id).get() ] # create dictionary of iota with key being their group label groups = {} for iota in data: label = int(iota["Label"]) if label not in groups: groups[label] = {} groups[label][iota["Key"]] = pickle.loads(iota["Value"]) # we know that dataframe labels are actually just their index value # so we can append these rows in order by simply looping through a range of their length and getting each one rows = [] for i in range(len(groups)): rows.append(groups[i]) # return frame return pd.DataFrame(rows)
def Devolver(self): self.__log.info("Devolver un libro") libro = Libro() print(f"\t ID\t Nombre\t ISBN\t Autor\t Editorial\t Estado") for obj in libro.all(): print(f"\t {obj.id}\t {obj.nombre}\t {obj.isbn}\t {obj.autors_id}\t {obj.editorial_id}\t {obj.estado_libro_id}") print("Escriba el id del libro") idlibro = input() DatabaseManager.table('libro').where('nombre', f'{obj.nombre}').update({'obj.estado_libro_id': 1}) libro.libros_id = idlibro libro.save() objMenus = init.Inicio() objMenus.MenuInicioMenu()
def dump(db_path): """Dump the DB contents to STDOUT, requiring only that the DB is a version that has an otus table in sqlite3 form (i.e. version 2 and 3 at least). """ sqlite_db = os.path.join(db_path, SequenceDatabase.SQLITE_DB_NAME) logging.debug("Connecting to DB {}".format(sqlite_db)) if not os.path.exists(sqlite_db): raise Exception( "SQLite3 database does not appear to exist in the SingleM database - perhaps it is the wrong version?" ) db = DatabaseManager( {'sqlite3': { 'driver': 'sqlite', 'database': sqlite_db }}) Model.set_connection_resolver(db) print "\t".join(OtuTable.DEFAULT_OUTPUT_FIELDS) for chunk in db.table('otus').chunk(1000): for entry in chunk: otu = OtuTableEntry() otu.marker = entry.marker otu.sample_name = entry.sample_name otu.sequence = entry.sequence otu.count = entry.num_hits otu.coverage = entry.coverage otu.taxonomy = entry.taxonomy print str(otu)
def dump(db_path): """Dump the DB contents to STDOUT, requiring only that the DB is a version that has an otus table in sqlite3 form (i.e. version 2 and 3 at least). """ sqlite_db = os.path.join(db_path, SequenceDatabase.SQLITE_DB_NAME) logging.debug("Connecting to DB {}".format(sqlite_db)) if not os.path.exists(sqlite_db): raise Exception("SQLite3 database does not appear to exist in the SingleM database - perhaps it is the wrong version?") db = DatabaseManager({ 'sqlite3': { 'driver': 'sqlite', 'database': sqlite_db }}) Model.set_connection_resolver(db) print "\t".join(OtuTable.DEFAULT_OUTPUT_FIELDS) for chunk in db.table('otus').chunk(1000): for entry in chunk: otu = OtuTableEntry() otu.marker = entry.marker otu.sample_name = entry.sample_name otu.sequence = entry.sequence otu.count = entry.num_hits otu.coverage = entry.coverage otu.taxonomy = entry.taxonomy print str(otu)
def index(): ts = time() st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S') db = DatabaseManager(app.config.get('DB_CONFIG')).connection() res = db.table('greetings').get(['message']).first() return render_template('index.html', message=res.get('message'), time=st, image_to_show=app.config.get('IMAGE_DEST_PATH'))
class Contractor(Process): def __init__(self, queue): super(Contractor, self).__init__() self.worker_control = Semaphore(MAX_CONTRACTOR_WORKERS) self.db = DatabaseManager(DATABASES) self.result_queue = queue def reset_processing_to_pending(self): self.db.table("tasks").where('processing_status', PROCESSING).update( {'processing_status': PENDING}) def run(self): print("*** Contractor online") print( "**** Resetting all halted tasks which status is processing into new pending" ) self.reset_processing_to_pending() while True: if self.worker_control.acquire(False): try: #get pending task task = self.db.table('tasks').where( 'processing_status', PENDING).first() if task: self.db.table("tasks").where('id', task.id).update( {'processing_status': PROCESSING}) worker = ContractorWorker(task, self.worker_control, self.result_queue) worker.start() else: self.worker_control.release() except Exception as err: print(err) self.worker_control.release() time.sleep(0.2)
class AppEloquent(): def __init__(self): self.configuration = {'mysql':{'driver':'mysql','host': 'localhost','database': 'atm3',\ 'user': '******','password': '', 'prefix': '' } } self.db = DatabaseManager(self.configuration) self.customer = self.db.table('customers') self.account = self.db.table('accounts') self.transaction = self.db.table('transactions') # # print(self.firstUser('id', 23)) # print(self.firstAccount('type', 2)) # print(self.statement('customer', 1)) def firstUser(self, identifier, data): return self.customer.where(identifier, data).first() # print(mir) def firstAccount(self, identifier, data): return self.account.where(identifier, data).first() def statement(self, identifier, data): return self.transaction.where(identifier, data).get()
class SimpleWorker(Process): def __init__(self,task_id,worker_control): super(SimpleWorker, self).__init__() self.task_id = task_id self.worker_control = worker_control self.db = DatabaseManager(DATABASES) def run(self): print "-- SIMPLE WORKER ONLINE : %s" % self.task_id t = self.db.table("tasks").where("task_id","=",self.task_id).first() if t is not None: try: target_url = t.target_url image_url = t.image_url screen_width = t.screen_width screen_height = t.screen_height image_detector = ImageDetector() status, image_match = image_detector.process(self.task_id,target_url,image_url,screen_width,screen_height) print status print image_match self.db.table("tasks").where("task_id","=",t.task_id).update({ "image_path" : image_match, "status": status, "processing_status": 2 }) except: self.db.table("tasks").where("task_id","=",t.task_id).update({ "processing_status": 3 }) self.worker_control.release()
def get_file(self, db: orator.DatabaseManager, filepath: Union[str, pathlib.Path, None] = None, readpath: Union[str, pathlib.Path, None] = None, md5: Union[str, None] = None, sha256: Union[str, None] = None) -> Union[str, None]: # enforce types checks.check_types(db, orator.DatabaseManager) checks.check_types(filepath, [str, pathlib.Path, type(None)]) checks.check_types(readpath, [str, pathlib.Path, type(None)]) checks.check_types(md5, [str, type(None)]) checks.check_types(sha256, [str, type(None)]) # enforce at least one parameter given assert filepath is not None or \ readpath is not None or \ md5 is not None or \ sha256 is not None, \ "Provide filepath, an fms provided readpath, or a file hash." # try to find the fileid if md5 is not None: table = db.table("File").where("MD5", "=", md5).get() elif sha256 is not None: table = db.table("File").where("SHA256", "=", sha256).get() elif readpath is not None: table = db.table("File").where("ReadPath", "=", readpath).get() else: md5 = tools.get_file_hash(filepath) table = db.table("File").where("MD5", "=", md5).get() # try catch exists found = [dict(item) for item in table] try: found = found[0] except IndexError: found = None return found
def _connect_to_sqlite(self, db): sqlite_db_path = db.sqlite_file if not os.path.exists(sqlite_db_path): raise Exception("Sqlite database not found at '%s', indicating that either the SingleM database was built with an out-dated SingleM version, or that the database is corrupt. Please generate a new database with the current version of SingleM.") logging.debug("Connecting to %s" % sqlite_db_path) dbm = DatabaseManager({ 'sqlite3': { 'driver': 'sqlite', 'database': sqlite_db_path }}) Model.set_connection_resolver(dbm) try: len(dbm.table('otus').limit(1).get()) except Exception as e: logging.error("Failure to extract any data from the otus table of the SQ Lite DB indicates this SingleM DB is either too old or is corrupt.") raise(e) try: len(dbm.table('clusters').limit(1).get()) except QueryException: logging.error("Failure to extract any data from the 'clusters' table indicates this SingleM DB is out-dated, and cannot be used with query implemented in this version of SingleM") sys.exit(1) return dbm
class OratorUserRepository(UserRepository): config: InitVar[Dict] db: DatabaseManager = field(init=False) def __post_init__(self, config: Dict): self.db = DatabaseManager(config) Model.set_connection_resolver(self.db) def register(self, user: DomainUser) -> None: orator_user = OratorUser() orator_user.id = user.user_id.value orator_user.name = user.user_name.value orator_user.age = user.user_age.value orator_user.save() def find_all(self) -> List[DomainUser]: orator_users = OratorUser.all() domain_users: List[DomainUser] = [] for orator_user in orator_users: domain_user = DomainUser(user_id=UserId.reconstruct( orator_user.id), user_name=UserName(orator_user.name), user_age=UserAge(orator_user.age)) domain_users.append(domain_user) return domain_users def find_by_name(self, name: str) -> Optional[DomainUser]: orator_user = self.db.table('users').where('name', name).first() if orator_user is None: return None return self._to_domain_user(orator_user) def find_by_id(self, user_id: UserId) -> Optional[DomainUser]: orator_user = OratorUser.find(user_id.value) if orator_user is None: return None return self._to_domain_user(orator_user) def _to_domain_user(self, orator_user: OratorUser) -> DomainUser: return DomainUser(user_id=UserId.reconstruct(orator_user.id), user_name=UserName(orator_user.name), user_age=UserAge(orator_user.age))
def on_get(self, req, resp): resp.set_header('Access-Control-Allow-Origin', '*') resp.set_header('Access-Control-Allow-Methods', 'GET') resp.set_header('Access-Control-Allow-Headers', 'Content-Type') # reading config config = configparser.ConfigParser() config.readfp(open('config.ini')) dbhost = config.get('database', 'host') dbdatabase = config.get('database', 'database') dbuser = config.get('database', 'user') dbpassword = config.get('database', 'password') prefix = config.get('database', 'prefix') wkhtmltopdf_path = config.get('wkhtmltopdf', 'path') baseurl = config.get('frontend', 'baseurl') #Setup DB config = { 'mysql': { 'driver': 'mysql', 'host': dbhost, 'database': dbdatabase, 'user': dbuser, 'password': dbpassword, 'prefix': '' } } db = DatabaseManager(config) Model.set_connection_resolver(db) try: key = req.params['key'] data = {"status": "success", "data": {}} except: resp.status = falcon.HTTP_404 return try: val = db.table(prefix + 'files').where('file_key', key).first() except: esp.status = falcon.HTTP_503 return try: resp.set_header("Content-Disposition", "attachment; filename=" + key + ".pdf") resp.content_type = "application/pdf" resp.data = val['file'] resp.status = falcon.HTTP_200 except: resp.status = falcon.HTTP_404
def setUp(self): config = self.manager.get_config() db = DatabaseManager(config) db.table('rooms').truncate()
class Datagenerator(): def __init__(self): # read config file required_fields = { "database": ["host", "database", "user", "password", "idfield", "textfield"] } self.config = configparser.ConfigParser() config.read('../settings_private.ini') for key, values in required_fields.items(): for value in values: if not config.has_option(key, value): raise Exception("Missing '" + str(value) + "' in section '" + str(key) + "' in settings.ini") def loadData(self): # generate data holding files d = pandas.DataFrame(columns=["id", "Sentiment", "Tweet"]) if not os.path.exists("../tweets_pos.csv"): d.to_csv("../tweets_pos.csv", index=False) if not os.path.exists("../tweets_neu.csv"): d.to_csv("../tweets_neu.csv", index=False) if not os.path.exists("../tweets_neg.csv"): d.to_csv("../tweets_neg.csv", index=False) pos = pandas.read_csv("../tweets_pos.csv") neu = pandas.read_csv("../tweets_neu.csv") neg = pandas.read_csv("../tweets_neg.csv") tweets_categorized = pos["id"].to_list() + neu["id"].to_list( ) + neg["id"].to_list() tweets = self.getTweets() tweets = [{ "id": id, "text": re.sub('<[^<]+?>', '', text) } for id, text in tweets.items() if id not in tweets_categorized and text[0:3] != "RT "] tweets_pos = {"id": [], "Sentiment": [], "Tweet": []} tweets_neu = {"id": [], "Sentiment": [], "Tweet": []} tweets_neg = {"id": [], "Sentiment": [], "Tweet": []} i = 0 for tweet in tweets: print("###### Tweet No. " + str(i) + "/" + str(len(tweets)) + " ######") print(tweet["text"]) print(colored("0, e, s to exit", "white")) print(colored("1 for positive", "green"), colored("2 for neutral", "yellow"), colored("3 for negative", "red")) key = msvcrt.getch() if chr(ord(key)) == "1": print("----") print(colored("Positiv", 'green')) tweets_pos["id"].append(tweet["id"]) tweets_pos["Sentiment"].append("positiv") tweets_pos["Tweet"].append(tweet["text"].replace( '\n', ' ').replace('\r', '').replace('"', "")) elif chr(ord(key)) == "2": print("----") print(colored("Neutral", 'yellow')) tweets_neu["id"].append(tweet["id"]) tweets_neu["Sentiment"].append("neutral") tweets_neu["Tweet"].append(tweet["text"].replace( '\n', ' ').replace('\r', '').replace('"', "")) elif chr(ord(key)) == "3": print("----") print(colored("Negativ", 'red')) tweets_neg["id"].append(tweet["id"]) tweets_neg["Sentiment"].append("negativ") tweets_neg["Tweet"].append(tweet["text"].replace( '\n', ' ').replace('\r', '').replace('"', "")) elif chr(ord(key)) in ["0", "e", "s"]: break i += 1 p = pandas.DataFrame.from_dict(tweets_pos) p.to_csv("../tweets_pos.csv", mode="a", index=False, header=False, quoting=csv.QUOTE_ALL) n = pandas.DataFrame.from_dict(tweets_neu) n.to_csv("../tweets_neu.csv", mode="a", index=False, header=False, quoting=csv.QUOTE_ALL) n = pandas.DataFrame.from_dict(tweets_neg) n.to_csv("../tweets_neg.csv", mode="a", index=False, header=False, quoting=csv.QUOTE_ALL) def getTweets(self): config = { 'mysql': { 'driver': 'mysql', 'host': self.config['database']['host'], 'database': self.config['database']['database'], 'user': self.config['database']['user'], 'password': self.config['database']['password'], 'prefix': '' } } self.db = DatabaseManager(config) tweets = self.db.table(self.config['database']['table']).where( self.config['database']['retweetfield'], '=', '0').lists(self.config['database']['textfield'], self.config['database']['idfield']) return tweets
'host': 'localhost', 'database': 'db_wo', 'user': '******', 'password': '******', 'prefix': '' } } db = DatabaseManager(config) results = db.select('SELECT * from tb_pesan_gedung where (now() > DATE_ADD(waktu_pesan, INTERVAL 2 HOUR)) and status = "pending" and status_pembayaran = "Belum Melakukan Pembayaran"') # print results for r in results: # print r db.table('tb_pesan_gedung').where('id_pesan', r['id_pesan']).update(status='expired') # query # SELECT # waktu_pesan, # now() as sekarang, # DATE_ADD(waktu_pesan, INTERVAL 2 HOUR) as expired, # if(now() > (DATE_ADD(waktu_pesan, INTERVAL 2 HOUR)), 'lebih','kurang') as jam_setelah # from tb_pesan_gedung # where # (DATE_ADD(NOW(), INTERVAL 2 HOUR) > waktu_pesan) # and # status = 'pending' #install orator #cara nya: pip install orator
def on_get(self, req, resp): resp.set_header('Access-Control-Allow-Origin', '*') resp.set_header('Access-Control-Allow-Methods', 'GET') resp.set_header('Access-Control-Allow-Headers', 'Content-Type') # reading config config = configparser.ConfigParser() config.readfp(open('config.ini')) dbhost = config.get('database', 'host') dbdatabase = config.get('database', 'database') dbuser = config.get('database', 'user') dbpassword = config.get('database', 'password') prefix = config.get('database', 'prefix') wkhtmltopdf_path = config.get('wkhtmltopdf', 'path') baseurl = config.get('frontend', 'baseurl') #Setup DB config = { 'mysql': { 'driver': 'mysql', 'host': dbhost, 'database': dbdatabase, 'user': dbuser, 'password': dbpassword, 'prefix': '' } } db = DatabaseManager(config) Model.set_connection_resolver(db) try: key = req.params['key'] data = {"status": "success", "data": {}} except: resp.status = falcon.HTTP_404 return try: val = db.table(prefix + 'files').where('file_key', key).count() except: esp.status = falcon.HTTP_503 return if val > 0: try: resp.body = json.dumps({ "status": "success", "data": { "file_status": "exists", "url": baseurl + "file?key=" + key } }) resp.status = falcon.HTTP_200 except: resp.status = falcon.HTTP_404 else: resp.body = json.dumps({ "status": "success", "data": { "file_status": "not exists" } })
class BatchWorker(Process): def __init__(self,task_id,worker_control): super(BatchWorker, self).__init__() self.task_id = task_id self.worker_control = worker_control self.db = DatabaseManager(DATABASES) def run(self): print "-- BATCH WORKER ONLINE : %s" % self.task_id image_detector = ImageDetector() tasks = self.db.table("tasks").where("task_id","=",self.task_id).order_by("id","asc").get() counter = 0 is_ok_to_continue = True for t in tasks: counter = counter + 1 try: if is_ok_to_continue: target_url = t.target_url image_url = t.image_url screen_width = t.screen_width screen_height = t.screen_height task_id = "%s_%s" % (self.task_id,counter) status, image_match = image_detector.process(task_id,target_url,image_url,screen_width,screen_height) print status print image_match self.db.table("tasks").where("task_id","=",t.task_id).update({ "image_path" : image_match, "status": status, "processing_status": 2 }) if status: is_ok_to_continue = False else: # at least one of the previous task has succeeded # the rest of the task will be set to DONE self.db.table("tasks").where("task_id","=",t.task_id).update({ "image_path" : "", "processing_status": 2 }) except: self.db.table("tasks").where("task_id","=",t.task_id).update({ "image_path" : "", "processing_status": 3 }) self.worker_control.release()
class Classifier: def __init__(self, train_test_split): self.split = train_test_split # read config file required_fields = {"database": ["host", "database", "user", "password", "idfield", "textfield"]} self.config = config = configparser.ConfigParser() config.read('../settings_private.ini') for key, values in required_fields.items(): for value in values: if not config.has_option(key, value): raise Exception("Missing '"+str(value)+"' in section '"+str(key)+"' in settings.ini") def train(self): self.pos = pandas.read_csv("../tweets_pos.csv") self.pos = self.pos["Tweet"].to_list() self.pos = list(set(self.pos)) self.neg = pandas.read_csv("../tweets_neg.csv") self.neg = self.neg["Tweet"].to_list() self.neg = list(set(self.neg)) X_train, X_test = self.buildFeatures() self.classifier = NaiveBayesClassifier.train(X_train) print("Accuracy of the classifier: ", nltk.classify.util.accuracy(self.classifier, X_test)) def extract_features(self, word_list): return dict([(word, True) for word in word_list]) def buildFeatures(self): pos_th = int(self.split*len(self.pos)) neg_th = int(self.split*len(self.pos)) feat_pos = [(self.extract_features(t.split()), 'Positive') for t in self.pos] feat_neg = [(self.extract_features(t.split()), 'Negative') for t in self.neg] return feat_pos[0:pos_th]+feat_neg[0:neg_th], feat_pos[pos_th:]+feat_neg[neg_th:] def predict(self, text): print("PREDICT: ", text) probdist = self.classifier.prob_classify(self.extract_features(str(text).split())) pred_sentiment = probdist.max() print("Predicted sentiment: ", pred_sentiment) print("Probability: ", round(probdist.prob(pred_sentiment), 2)) return pred_sentiment, round(probdist.prob(pred_sentiment), 2) def mostImportant(self): print("Top ten most informative words: ") for item in self.classifier.most_informative_features()[:10]: print(item[0]) def save(self, filename): import pickle f = open(filename, 'wb') pickle.dump(self.classifier, f) f.close() def load(self, filename): import pickle f = open(filename, 'rb') self.classifier = pickle.load(f) f.close() # read tweets from a database def getTweets(self): config = { 'mysql': { 'driver': 'mysql', 'host': self.config['database']['host'], 'database': self.config['database']['database'], 'user': self.config['database']['user'], 'password': self.config['database']['password'], 'prefix': '' } } self.db = DatabaseManager(config) tweets = self.db.table(self.config['database']['table']).where(self.config['database']['retweetfield'], '=', '0').lists(self.config['database']['textfield']) return tweets def predictFromDb(self): tweets = self.getTweets() tweets = [re.sub('<[^<]+?>', '', text) for text in tweets if text[0:3] != "RT "] preds = [] for tweet in tweets: pred, acc = self.predict(tweet.replace('\n', ' ').replace('\r', '').replace('"', "")) if float(acc) < 0.65: preds.append("Unsure") print("Unsure: ", acc) else: preds.append(pred) return preds
class Driver(Model): __table__ = 'drivers' pass print("Buscando motoristas...") contacts = wppDriver.get_contacts() print(str(len(contacts)) + " Contatos") drivers = Driver.where_raw('options_stage = 0').get() phones_that_received = map(lambda x: x.phone, drivers) new_contacts = [a for a in contacts if a.id not in phones_that_received] print(str(len(new_contacts)) + "Novos contatos esperando mensagens") print(datetime.datetime.now()) iteraction = 0 for driver in new_contacts: print("iteração: "+ str(iteraction)) iteraction += 1 if iteraction == 1000: print("chegou nos mil") raise "error" try: db.table('drivers').insert(phone=driver.id, name=driver.name, created_at=datetime.datetime.now(), updated_at=datetime.datetime.now()) except: print("erro no usuário "+ driver.id) print(datetime.datetime.now())
def on_post(self, req, resp): resp.set_header('Access-Control-Allow-Origin', '*') resp.set_header('Access-Control-Allow-Methods', 'GET') resp.set_header('Access-Control-Allow-Headers', 'Content-Type') # reading config config = configparser.ConfigParser() config.readfp(open('config.ini')) dbhost = config.get('database', 'host') dbdatabase = config.get('database', 'database') dbuser = config.get('database', 'user') dbpassword = config.get('database', 'password') prefix = config.get('database', 'prefix') wkhtmltopdf_path = config.get('wkhtmltopdf', 'path') baseurl = config.get('frontend', 'baseurl') #Setup DB config = { 'mysql': { 'driver': 'mysql', 'host': dbhost, 'database': dbdatabase, 'user': dbuser, 'password': dbpassword, 'prefix': '' } } #TODO create tables db = DatabaseManager(config) Model.set_connection_resolver(db) data = {"status": "failure", "data": {}} try: title = req.params['title'] data = {"status": "success", "data": {}} except: title = '' try: md = req.params['markdown'] data = {"status": "success", "data": {}} except: data = {"status": "failure", "data": {}} md = False if md != False: md = head + convertMarkdownToHtml(md.replace("/n", "<br>")) #md += "<br><div class='end'>www.piratenpartei.de</div>" key = datetime.datetime.now().strftime( "%Y-%m-%d_") + randomString() font_config = FontConfiguration() file_ = HTML(string=md).write_pdf(font_config=font_config) try: db.table(prefix + 'files').insert({ 'file_key': key, 'file': file_ }) data = { "status": "success", "data": { "key": key, "url": baseurl + key } } except Exception as e: print(e) data = {"status": "failure", "data": {}} resp.status = falcon.HTTP_503 return else: data = {"status": "failure", "data": {}} resp.body = json.dumps(data)
if __name__ == '__main__': print "PROCESSER RUNNING" print "#CPU COUNT : %s" % psutil.cpu_count() worker_control = Semaphore(psutil.cpu_count()) while True: if worker_control.acquire(False): db.reconnect() t = db.table("tasks").where("processing_status","=",0).first() if t is not None: task_id = t.task_id db.table("tasks").where("task_id","like",task_id).update({'processing_status': 1}) num_tasks = db.table("tasks").where("task_id","like",task_id).count() print num_tasks if num_tasks > 1: w = BatchWorker(task_id,worker_control) w.start() else: w = SimpleWorker(task_id,worker_control)
parser = argparse.ArgumentParser(description='Update new database from old') parser.add_argument('database', help='Database file') args = vars(parser.parse_args()) # Connects to a sqlite database used to store comment ids. print('Connecting to database...') try: connection = sqlite3.connect(args['database']) connection.row_factory = sqlite3.Row cursor = connection.cursor() print('Connected to database!') except: print('Connection to database failed.',sys.exc_info()[0]) exit() cursor.execute('select id,utc_time from comments where utc_time > ' + str(time.time() - 31*24*60*60)) number = 0 inserts = list() print("Creating list") for cid in cursor: if (number > 0) and (number % 250) == 0: print("Bulk insertion %d" % (number)) db.table('comments').insert(inserts) inserts = list() inserts.append({'commentId': cid[0], 'utcTime': cid[1]}) number += 1 print("Bulk insertion") db.table('comments').insert(inserts)
from orator import DatabaseManager import logging logging.basicConfig(level=logging.DEBUG) # https://orator-orm.com/docs/0.9/query_builder.html # https://orator-orm.com/docs/0.9/basic_usage.html#query-logging config = { "sqlite3": {"driver": "sqlite", "database": "examples.db", "log_queries": True} } db = DatabaseManager(config) with db.transaction(): user = db.table("users").first() print(user) with db.transaction(): user = db.table("users").select("id", "name").first() print(user) print("----------------------------------------") for users in db.table("users").select("id", "name").chunk(100): for user in users: print(user)
def RegistroLibros(self): self.__log.info("Ingresando al Registro de Libros") opcionesRegisLibros = { "\t- Registrar Libro": 1, "\t- Listar Libros": 2, "\t- Eliminar Libros": 3 } MenuRegisLibros = Menu("Menu Modulo Libros", opcionesRegisLibros) regisLibros = MenuRegisLibros.mostrarMenu() menuRegisLib = True while menuRegisLib: if (regisLibros == 1): nuevoLibro = Libro() nombreLibro = input("escriba el nombre del Libro \n") isbnLibro = input("escriba el ISBN del libro \n") autores = Autor() print(f"\t ID\t Nombre\t Correo") for obj in autores.all(): print(f"\t {obj.id}\t {obj.nombre}\t {obj.correo}") print("Escriba el id del Autor") autor_idLibro = input() editoriales = Editorial() print(f"\t ID\t Nombre") for obj in editoriales.all(): print(f"\t {obj.id}\t {obj.nombre}") print("Escriba el id de la editorial") editorialLibro = input() nuevoLibro.nombre = nombreLibro nuevoLibro.isbn = isbnLibro nuevoLibro.autor_id = autor_idLibro nuevoLibro.estado_libro_id = editorialLibro nuevoLibro.save() menuRegisLib = False elif (regisLibros == 2): libros = Libro() print(f"ID\t NOMBRE\t ISBN\t AUTOR\t EDITORIAL\t ESTADO") for obj in libros.all(): print( f"\t {obj.id}\t {obj.nombre}\t {obj.ISBN}\t {obj.autors_id}\t {obj.editorial_id}\t {obj.estado_libro_id}" ) print("Escriba el id de la editorial") editorialLibro = input() input("Regresar???") menuRegisLib = False program.utils.Menu("Menu Modulo Libros", regisLibros) elif (regisLibros == 3): libros = Libro() print(f"ID\t NOMBRE\t ISBN\t AUTOR\t EDITORIAL\t ESTADO") for obj in libros.all(): print( f"\t {obj.id}\t {obj.nombre}\t {obj.isbn}\t {obj.autors_id}\t {obj.editorial_id}\t {obj.estado_libro_id}" ) print("Escriba el id del libro que se desea eliminar: \n") id = input() DatabaseManager.table('libros').get() DatabaseManager.table('libros').where('id', '=', f'{id}').delete() menuRegisLib = False elif (regisLibros == 9): __log = log("Saliendo") break
return str(o) def dumps(body): return json.dumps(body or {}, default=converter) def new_error(message, code): return { 'code': code, 'message': message, } JWT_SECRET = db.table('secret').where('key', 'JWT').pluck('body') def jwt_encode(user_id): encoded = jwt.encode({'user_id': user_id}, JWT_SECRET, algorithm='HS256') return encoded.decode('utf8') def jwt_decode(raw): token = raw if raw.startswith('Bearer'): token = raw[7:] try:
with urllib.request.urlopen(api_endpoint) as url: json_data = json.loads(url.read().decode()) # Set field sizes - we know some are longer than 255 print(f"Setting field sizes") for field in data_fields.keys(): # Max of length of values of entries in the array seq = [len(x[field]) for x in json_data if x[field]] if max(seq) > 255: data_fields[field]["size"] = max(seq) else: data_fields[field]["size"] = 255 # Create a table print("Creating DB table named 'data'") with schema.create('data') as table: for field in data_fields.keys(): if data_fields[field]["type"] == "string": table.string(field, data_fields[field]["size"]).nullable() if data_fields[field]["type"] == "integer": table.integer(field).nullable() if data_fields[field]["type"] == "datetime": table.string(field).nullable() # Fire in the data print("Writing records") for record in json_data: db.table('data').insert(record) print("Complete!")
comment.reply(commandResponse) girmRequestsToFind = re.findall(r'\[\s*girm\s*([\d\-,\s]+)\s*\\?\](?im)(?!\()', comment.body) if girmRequestsToFind: requestIsValid,commandResponse = girmResponse.getResponse(girmRequestsToFind) if requestIsValid: try: comment.reply(commandResponse) except praw.exceptions.APIException as e: if e.error_type == "RATELIMIT": logger.warning("Sleeping 90 seconds due to RateLimitExceed ") sleep(90) comment.reply(commandResponse) try: # db.connection().enable_query_log() logger.info("Cleaning database of old comments") db.table('comments').where('utcTime', '<', time.time() - 31*24*60*60).delete() dbComment = Comment() dbComment.commentId = comment.id dbComment.utcTime = time.time() dbComment.save() logger.info("Inserting comment %s by user %s into database" % (comment.id, comment.author.name)) except: logger.critical("Database insert for comment %s by user %s failed." % (comment.id, comment.author.name),sys.exc_info()[0],"x") exit() except requests.exceptions.HTTPError: logger.error("HTTP Error: waiting 5 minutes to retry"+sys.exc_info()[0],"e") sleep(5*60)