def _connect_to_sqlite(self, db):
sqlite_db_path = db.sqlite_file
if not os.path.exists(sqlite_db_path):
raise Exception(
"Sqlite database not found at '%s', indicating that either the SingleM database was built with an out-dated SingleM version, or that the database is corrupt. Please generate a new database with the current version of SingleM."
)
logging.debug("Connecting to %s" % sqlite_db_path)
dbm = DatabaseManager(
{'sqlite3': {
'driver': 'sqlite',
'database': sqlite_db_path
}})
Model.set_connection_resolver(dbm)
try:
len(dbm.table('otus').limit(1).get())
except Exception as e:
logging.error(
"Failure to extract any data from the otus table of the SQ Lite DB indicates this SingleM DB is either too old or is corrupt."
)
raise (e)
try:
len(dbm.table('clusters').limit(1).get())
except QueryException:
logging.error(
"Failure to extract any data from the 'clusters' table indicates this SingleM DB is out-dated, and cannot be used with query implemented in this version of SingleM"
)
sys.exit(1)
return dbm
def load_data():
try:
db = DatabaseManager(app.config.get('DB_CONFIG')).connection()
if not db.table('greetings').get(['message']).first():
db.table('greetings').insert({'message': 'Hello world!'})
except Exception as e:
app.logger.error('Unable to connect to db: %s' % e)
class WriterWorker(Thread):
def __init__(self, task, worker_control):
Thread.__init__(self)
self.task = task
self.worker_control = worker_control
self.db = DatabaseManager(DATABASES)
def update_task_status(self, params):
print("- Updating processing_status for task id %s with processing_status of %s into the database" % (params['id'], params['processing_status']))
self.db.table("tasks").where("id", params['id']).update({"ipfs_hash": params["ipfs_hash"], "processing_status": params['processing_status']})
print("-- Done updating the task-id %s into the database" % params['id'])
def run(self):
print("Writer worker online for task-id: %s" % self.task["id"])
try:
self.update_task_status(self.task)
except Exception as err:
print(err)
self.worker_control.release()
def get_or_create_file(self,
db: orator.DatabaseManager,
filepath: Union[str, pathlib.Path],
metadata: Union[str, dict, None] = None) -> dict:
# enforce types
checks.check_types(db, orator.DatabaseManager)
checks.check_types(filepath, [str, pathlib.Path])
checks.check_types(metadata, [str, dict, type(None)])
# convert types
filepath = pathlib.Path(filepath)
if isinstance(metadata, dict):
metadata = str(dict)
# check file exists
checks.check_file_exists(filepath)
# check exists
md5 = tools.get_file_hash(filepath)
sha256 = tools.get_file_hash(filepath, hashlib.sha256)
file_info = self.get_file(db=db, md5=md5)
# return if found
if file_info is not None:
return file_info
name = "fms_" + md5
# create if not
with tools.suppress_prints():
self._build_file_as_package(filepath, name)
# import string
read_pkg = importlib.import_module(name="quilt.data." +
self.storage_user + "." + name)
# file info dict
file_info = {
"FileId": str(uuid.uuid4()),
"OriginalFilepath": str(filepath),
"FileType": filepath.suffix[1:],
"ReadPath": read_pkg.load(),
"MD5": md5,
"SHA256": sha256,
"Metadata": metadata,
"Created": datetime.utcnow()
}
# insert
db.table("File").insert(file_info)
return file_info
def Alquilar(self):
self.__log.info("Alquilar un libro")
estadolibro = EstadoLibro()
print(f"\t ID\t Descripcion")
for obj in estadolibro.all():
print(f"\t {obj.id}\t {obj.descripcion}")
if obj.descripcion == 1:
libroporalquilar = Prestamo()
cliente = User()
print(f"\t ID\t Nombre\t CORREO\t DOCUMENTO\t ESTADO")
for obj in cliente.all():
print(f"\t {obj.id}\t {obj.nombre}\t {obj.correo}\t {obj.documento}\t {obj.estado_user_id}")
print("Escriba el id del cliente")
cliente = input()
libro = Libro()
print(f"\t ID\t Nombre\t ISBN\t Autor\t Editorial\t Estado")
for obj in libro.all():
print(f"\t {obj.id}\t {obj.nombre}\t {obj.isbn}\t {obj.autors_id}\t {obj.editorial_id}\t {obj.estado_libro_id}")
print("Escriba el id del libro")
libro = input()
DatabaseManager.table('libro').where('nombre', f'{obj.nombre}').update({'obj.estado_libro_id': 3})
biblioteca = Biblioteca()
print(f"\t ID\t Nombre\t DIRECCION")
for obj in biblioteca.all():
print(f"\t {obj.id}\t {obj.nombre}\t {obj.direccion}")
print("Escriba el id de la biblioteca")
biblioteca = input()
libroporalquilar.users_id = cliente
libroporalquilar.libros_id = libro
libroporalquilar.bibliotecas_id = biblioteca
libroporalquilar.save()
objMenus = init.Inicio()
objMenus.MenuInicioMenu()
elif obj.descripcion == 2:
print("Lo sentimos el libro que busca esta reservado")
objMenus = init.Inicio()
objMenus.MenuInicioMenu()
elif obj.descripcion == 2:
print("Lo sentimos el libro que busca no se encuentra disponible")
objMenus = init.Inicio()
objMenus.MenuInicioMenu()
def RegistroEditorial(self):
menuRegisEdit = True
__log = log("Ingresando al Registro de las editoriales")
self.__log.info("Ingresando al Registro de las editoriales")
opcionesRegisEditorial = {
"Registrar Editorial": 1,
"Listar Editoriales": 2,
"Eliminar Editoriales": 3
}
MenuRegisEditorial = Menu("Menu Modulo Libros", opcionesRegisEditorial)
regisEditoriales = MenuRegisEditorial.mostrarMenu()
menuRegisEdit = True
while menuRegisEdit:
if (regisEditoriales == 1):
nuevaEditorial = Editorial()
nombreEditorial = input("escriba el nombre de la editorial \n")
nuevaEditorial.nombre = nombreEditorial
nuevaEditorial.save()
menuRegisEdit = False
elif (regisEditoriales == 2):
editorial = Editorial()
print(f"\t ID\t NOMBRE")
for obj in editorial.all():
print(f"\t {obj.id}\t {obj.nombre}")
input("Regresar???")
program.utils.Menu("Menu Modulo Editoriales", regisEditoriales)
elif (regisEditoriales == 3):
editorial = Editorial()
print(f"\t ID\t NOMBRE")
for obj in editorial.all():
print(f"\t {obj.id}\t {obj.nombre}")
print(
"Escriba el id de la editorial que se desea eliminar: \n")
id = input()
DatabaseManager.table('editorial').get()
DatabaseManager.table('editorial').where('id', '=',
f'{id}').delete()
elif (regisEditoriales == 9):
__log = log("Saliendo")
break
class Loader(Process):
def __init__(self):
super(Loader, self).__init__()
self.db = DatabaseManager(DATABASES)
def run(self):
print("*** Loader online")
images = get_all_images() ##get all images in list and put that images in the database and set to new PENDING
if images:
for image in images:
print("Inserting image file-path %s into the database" %image)
self.db.table("tasks").insert({'image_path': image})
class MainOperate:
def __init__(self):
self.db = DatabaseManager(config=CONFIG).connection()
def one_mysql_link(self):
"""
根据offset limit 查询数据
:return:
"""
result = self.db.table("corpus").offset(2).limit(5).get()
# 对于多条件查询
db_data = self.db.table("corpus").where('type', 'all').where_between(
'age', [15, 25]).get()
for num, data in enumerate(result):
# 根据字段名获取每一条信息
logging.info(data.get("url"))
return result
def two_add_column(self):
"""
给已存在的数据表增加一列数据
:return:
"""
new_data = {
"title": "测试标题",
"url": "https://www.baidu.com",
"content": "测试正文",
"published": "2019-11-02 12:16:01",
}
result = self.db.table("corpus").insert(new_data)
logging.info(result)
def three_update_data(self):
"""
更新数据库操作
:return:
"""
corpus = self.db.table("corpus").where('id', '1').update(
{'published': '2019-12-03 09:25:01'})
logging.info(corpus)
def four_delete_data(self):
"""
根据id来删除一行数据
:return:
"""
delete_data = self.db.table("corpus").delete('1899')
logging.info(delete_data)
def reconstruct(db: orator.DatabaseManager, ds_info: "DatasetInfo",
fms: FMSInterface) -> pd.DataFrame:
# get all iota that match
data = [
dict(r) for r in db.table("Iota").
join("IotaGroup", "IotaGroup.IotaId", "=", "Iota.IotaId").join(
"GroupDataset", "GroupDataset.GroupId", "=", "IotaGroup.GroupId"
).join("Dataset", "GroupDataset.DatasetId", "=", "Dataset.DatasetId"
).where("Dataset.DatasetId", "=", ds_info.id).get()
]
# create dictionary of iota with key being their group label
groups = {}
for iota in data:
label = int(iota["Label"])
if label not in groups:
groups[label] = {}
groups[label][iota["Key"]] = pickle.loads(iota["Value"])
# we know that dataframe labels are actually just their index value
# so we can append these rows in order by simply looping through a range of their length and getting each one
rows = []
for i in range(len(groups)):
rows.append(groups[i])
# return frame
return pd.DataFrame(rows)
def Devolver(self):
self.__log.info("Devolver un libro")
libro = Libro()
print(f"\t ID\t Nombre\t ISBN\t Autor\t Editorial\t Estado")
for obj in libro.all():
print(f"\t {obj.id}\t {obj.nombre}\t {obj.isbn}\t {obj.autors_id}\t {obj.editorial_id}\t {obj.estado_libro_id}")
print("Escriba el id del libro")
idlibro = input()
DatabaseManager.table('libro').where('nombre', f'{obj.nombre}').update({'obj.estado_libro_id': 1})
libro.libros_id = idlibro
libro.save()
objMenus = init.Inicio()
objMenus.MenuInicioMenu()
def dump(db_path):
"""Dump the DB contents to STDOUT, requiring only that the DB is a version that
has an otus table in sqlite3 form (i.e. version 2 and 3 at least).
"""
sqlite_db = os.path.join(db_path, SequenceDatabase.SQLITE_DB_NAME)
logging.debug("Connecting to DB {}".format(sqlite_db))
if not os.path.exists(sqlite_db):
raise Exception(
"SQLite3 database does not appear to exist in the SingleM database - perhaps it is the wrong version?"
)
db = DatabaseManager(
{'sqlite3': {
'driver': 'sqlite',
'database': sqlite_db
}})
Model.set_connection_resolver(db)
print "\t".join(OtuTable.DEFAULT_OUTPUT_FIELDS)
for chunk in db.table('otus').chunk(1000):
for entry in chunk:
otu = OtuTableEntry()
otu.marker = entry.marker
otu.sample_name = entry.sample_name
otu.sequence = entry.sequence
otu.count = entry.num_hits
otu.coverage = entry.coverage
otu.taxonomy = entry.taxonomy
print str(otu)
def dump(db_path):
"""Dump the DB contents to STDOUT, requiring only that the DB is a version that
has an otus table in sqlite3 form (i.e. version 2 and 3 at least).
"""
sqlite_db = os.path.join(db_path, SequenceDatabase.SQLITE_DB_NAME)
logging.debug("Connecting to DB {}".format(sqlite_db))
if not os.path.exists(sqlite_db):
raise Exception("SQLite3 database does not appear to exist in the SingleM database - perhaps it is the wrong version?")
db = DatabaseManager({
'sqlite3': {
'driver': 'sqlite',
'database': sqlite_db
}})
Model.set_connection_resolver(db)
print "\t".join(OtuTable.DEFAULT_OUTPUT_FIELDS)
for chunk in db.table('otus').chunk(1000):
for entry in chunk:
otu = OtuTableEntry()
otu.marker = entry.marker
otu.sample_name = entry.sample_name
otu.sequence = entry.sequence
otu.count = entry.num_hits
otu.coverage = entry.coverage
otu.taxonomy = entry.taxonomy
print str(otu)
def index():
ts = time()
st = datetime.datetime.fromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
db = DatabaseManager(app.config.get('DB_CONFIG')).connection()
res = db.table('greetings').get(['message']).first()
return render_template('index.html',
message=res.get('message'),
time=st,
image_to_show=app.config.get('IMAGE_DEST_PATH'))
class Contractor(Process):
def __init__(self, queue):
super(Contractor, self).__init__()
self.worker_control = Semaphore(MAX_CONTRACTOR_WORKERS)
self.db = DatabaseManager(DATABASES)
self.result_queue = queue
def reset_processing_to_pending(self):
self.db.table("tasks").where('processing_status', PROCESSING).update(
{'processing_status': PENDING})
def run(self):
print("*** Contractor online")
print(
"**** Resetting all halted tasks which status is processing into new pending"
)
self.reset_processing_to_pending()
while True:
if self.worker_control.acquire(False):
try:
#get pending task
task = self.db.table('tasks').where(
'processing_status', PENDING).first()
if task:
self.db.table("tasks").where('id', task.id).update(
{'processing_status': PROCESSING})
worker = ContractorWorker(task, self.worker_control,
self.result_queue)
worker.start()
else:
self.worker_control.release()
except Exception as err:
print(err)
self.worker_control.release()
time.sleep(0.2)
class AppEloquent():
def __init__(self):
self.configuration = {'mysql':{'driver':'mysql','host': 'localhost','database': 'atm3',\
'user': '******','password': '', 'prefix': '' } }
self.db = DatabaseManager(self.configuration)
self.customer = self.db.table('customers')
self.account = self.db.table('accounts')
self.transaction = self.db.table('transactions')
# # print(self.firstUser('id', 23))
# print(self.firstAccount('type', 2))
# print(self.statement('customer', 1))
def firstUser(self, identifier, data):
return self.customer.where(identifier, data).first()
# print(mir)
def firstAccount(self, identifier, data):
return self.account.where(identifier, data).first()
def statement(self, identifier, data):
return self.transaction.where(identifier, data).get()
class SimpleWorker(Process):
def __init__(self,task_id,worker_control):
super(SimpleWorker, self).__init__()
self.task_id = task_id
self.worker_control = worker_control
self.db = DatabaseManager(DATABASES)
def run(self):
print "-- SIMPLE WORKER ONLINE : %s" % self.task_id
t = self.db.table("tasks").where("task_id","=",self.task_id).first()
if t is not None:
try:
target_url = t.target_url
image_url = t.image_url
screen_width = t.screen_width
screen_height = t.screen_height
image_detector = ImageDetector()
status, image_match = image_detector.process(self.task_id,target_url,image_url,screen_width,screen_height)
print status
print image_match
self.db.table("tasks").where("task_id","=",t.task_id).update({
"image_path" : image_match,
"status": status,
"processing_status": 2
})
except:
self.db.table("tasks").where("task_id","=",t.task_id).update({
"processing_status": 3
})
self.worker_control.release()
def get_file(self,
db: orator.DatabaseManager,
filepath: Union[str, pathlib.Path, None] = None,
readpath: Union[str, pathlib.Path, None] = None,
md5: Union[str, None] = None,
sha256: Union[str, None] = None) -> Union[str, None]:
# enforce types
checks.check_types(db, orator.DatabaseManager)
checks.check_types(filepath, [str, pathlib.Path, type(None)])
checks.check_types(readpath, [str, pathlib.Path, type(None)])
checks.check_types(md5, [str, type(None)])
checks.check_types(sha256, [str, type(None)])
# enforce at least one parameter given
assert filepath is not None or \
readpath is not None or \
md5 is not None or \
sha256 is not None, \
"Provide filepath, an fms provided readpath, or a file hash."
# try to find the fileid
if md5 is not None:
table = db.table("File").where("MD5", "=", md5).get()
elif sha256 is not None:
table = db.table("File").where("SHA256", "=", sha256).get()
elif readpath is not None:
table = db.table("File").where("ReadPath", "=", readpath).get()
else:
md5 = tools.get_file_hash(filepath)
table = db.table("File").where("MD5", "=", md5).get()
# try catch exists
found = [dict(item) for item in table]
try:
found = found[0]
except IndexError:
found = None
return found
def _connect_to_sqlite(self, db):
sqlite_db_path = db.sqlite_file
if not os.path.exists(sqlite_db_path):
raise Exception("Sqlite database not found at '%s', indicating that either the SingleM database was built with an out-dated SingleM version, or that the database is corrupt. Please generate a new database with the current version of SingleM.")
logging.debug("Connecting to %s" % sqlite_db_path)
dbm = DatabaseManager({
'sqlite3': {
'driver': 'sqlite',
'database': sqlite_db_path
}})
Model.set_connection_resolver(dbm)
try:
len(dbm.table('otus').limit(1).get())
except Exception as e:
logging.error("Failure to extract any data from the otus table of the SQ Lite DB indicates this SingleM DB is either too old or is corrupt.")
raise(e)
try:
len(dbm.table('clusters').limit(1).get())
except QueryException:
logging.error("Failure to extract any data from the 'clusters' table indicates this SingleM DB is out-dated, and cannot be used with query implemented in this version of SingleM")
sys.exit(1)
return dbm
class OratorUserRepository(UserRepository):
config: InitVar[Dict]
db: DatabaseManager = field(init=False)
def __post_init__(self, config: Dict):
self.db = DatabaseManager(config)
Model.set_connection_resolver(self.db)
def register(self, user: DomainUser) -> None:
orator_user = OratorUser()
orator_user.id = user.user_id.value
orator_user.name = user.user_name.value
orator_user.age = user.user_age.value
orator_user.save()
def find_all(self) -> List[DomainUser]:
orator_users = OratorUser.all()
domain_users: List[DomainUser] = []
for orator_user in orator_users:
domain_user = DomainUser(user_id=UserId.reconstruct(
orator_user.id),
user_name=UserName(orator_user.name),
user_age=UserAge(orator_user.age))
domain_users.append(domain_user)
return domain_users
def find_by_name(self, name: str) -> Optional[DomainUser]:
orator_user = self.db.table('users').where('name', name).first()
if orator_user is None:
return None
return self._to_domain_user(orator_user)
def find_by_id(self, user_id: UserId) -> Optional[DomainUser]:
orator_user = OratorUser.find(user_id.value)
if orator_user is None:
return None
return self._to_domain_user(orator_user)
def _to_domain_user(self, orator_user: OratorUser) -> DomainUser:
return DomainUser(user_id=UserId.reconstruct(orator_user.id),
user_name=UserName(orator_user.name),
user_age=UserAge(orator_user.age))
def on_get(self, req, resp):
resp.set_header('Access-Control-Allow-Origin', '*')
resp.set_header('Access-Control-Allow-Methods', 'GET')
resp.set_header('Access-Control-Allow-Headers', 'Content-Type')
# reading config
config = configparser.ConfigParser()
config.readfp(open('config.ini'))
dbhost = config.get('database', 'host')
dbdatabase = config.get('database', 'database')
dbuser = config.get('database', 'user')
dbpassword = config.get('database', 'password')
prefix = config.get('database', 'prefix')
wkhtmltopdf_path = config.get('wkhtmltopdf', 'path')
baseurl = config.get('frontend', 'baseurl')
#Setup DB
config = {
'mysql': {
'driver': 'mysql',
'host': dbhost,
'database': dbdatabase,
'user': dbuser,
'password': dbpassword,
'prefix': ''
}
}
db = DatabaseManager(config)
Model.set_connection_resolver(db)
try:
key = req.params['key']
data = {"status": "success", "data": {}}
except:
resp.status = falcon.HTTP_404
return
try:
val = db.table(prefix + 'files').where('file_key', key).first()
except:
esp.status = falcon.HTTP_503
return
try:
resp.set_header("Content-Disposition",
"attachment; filename=" + key + ".pdf")
resp.content_type = "application/pdf"
resp.data = val['file']
resp.status = falcon.HTTP_200
except:
resp.status = falcon.HTTP_404
def setUp(self):
config = self.manager.get_config()
db = DatabaseManager(config)
db.table('rooms').truncate()
class Datagenerator():
def __init__(self):
# read config file
required_fields = {
"database":
["host", "database", "user", "password", "idfield", "textfield"]
}
self.config = configparser.ConfigParser()
config.read('../settings_private.ini')
for key, values in required_fields.items():
for value in values:
if not config.has_option(key, value):
raise Exception("Missing '" + str(value) +
"' in section '" + str(key) +
"' in settings.ini")
def loadData(self):
# generate data holding files
d = pandas.DataFrame(columns=["id", "Sentiment", "Tweet"])
if not os.path.exists("../tweets_pos.csv"):
d.to_csv("../tweets_pos.csv", index=False)
if not os.path.exists("../tweets_neu.csv"):
d.to_csv("../tweets_neu.csv", index=False)
if not os.path.exists("../tweets_neg.csv"):
d.to_csv("../tweets_neg.csv", index=False)
pos = pandas.read_csv("../tweets_pos.csv")
neu = pandas.read_csv("../tweets_neu.csv")
neg = pandas.read_csv("../tweets_neg.csv")
tweets_categorized = pos["id"].to_list() + neu["id"].to_list(
) + neg["id"].to_list()
tweets = self.getTweets()
tweets = [{
"id": id,
"text": re.sub('<[^<]+?>', '', text)
} for id, text in tweets.items()
if id not in tweets_categorized and text[0:3] != "RT "]
tweets_pos = {"id": [], "Sentiment": [], "Tweet": []}
tweets_neu = {"id": [], "Sentiment": [], "Tweet": []}
tweets_neg = {"id": [], "Sentiment": [], "Tweet": []}
i = 0
for tweet in tweets:
print("###### Tweet No. " + str(i) + "/" + str(len(tweets)) +
" ######")
print(tweet["text"])
print(colored("0, e, s to exit", "white"))
print(colored("1 for positive", "green"),
colored("2 for neutral", "yellow"),
colored("3 for negative", "red"))
key = msvcrt.getch()
if chr(ord(key)) == "1":
print("----")
print(colored("Positiv", 'green'))
tweets_pos["id"].append(tweet["id"])
tweets_pos["Sentiment"].append("positiv")
tweets_pos["Tweet"].append(tweet["text"].replace(
'\n', ' ').replace('\r', '').replace('"', ""))
elif chr(ord(key)) == "2":
print("----")
print(colored("Neutral", 'yellow'))
tweets_neu["id"].append(tweet["id"])
tweets_neu["Sentiment"].append("neutral")
tweets_neu["Tweet"].append(tweet["text"].replace(
'\n', ' ').replace('\r', '').replace('"', ""))
elif chr(ord(key)) == "3":
print("----")
print(colored("Negativ", 'red'))
tweets_neg["id"].append(tweet["id"])
tweets_neg["Sentiment"].append("negativ")
tweets_neg["Tweet"].append(tweet["text"].replace(
'\n', ' ').replace('\r', '').replace('"', ""))
elif chr(ord(key)) in ["0", "e", "s"]:
break
i += 1
p = pandas.DataFrame.from_dict(tweets_pos)
p.to_csv("../tweets_pos.csv",
mode="a",
index=False,
header=False,
quoting=csv.QUOTE_ALL)
n = pandas.DataFrame.from_dict(tweets_neu)
n.to_csv("../tweets_neu.csv",
mode="a",
index=False,
header=False,
quoting=csv.QUOTE_ALL)
n = pandas.DataFrame.from_dict(tweets_neg)
n.to_csv("../tweets_neg.csv",
mode="a",
index=False,
header=False,
quoting=csv.QUOTE_ALL)
def getTweets(self):
config = {
'mysql': {
'driver': 'mysql',
'host': self.config['database']['host'],
'database': self.config['database']['database'],
'user': self.config['database']['user'],
'password': self.config['database']['password'],
'prefix': ''
}
}
self.db = DatabaseManager(config)
tweets = self.db.table(self.config['database']['table']).where(
self.config['database']['retweetfield'], '=',
'0').lists(self.config['database']['textfield'],
self.config['database']['idfield'])
return tweets
'host': 'localhost',
'database': 'db_wo',
'user': '******',
'password': '******',
'prefix': ''
}
}
db = DatabaseManager(config)
results = db.select('SELECT * from tb_pesan_gedung where (now() > DATE_ADD(waktu_pesan, INTERVAL 2 HOUR)) and status = "pending" and status_pembayaran = "Belum Melakukan Pembayaran"')
# print results
for r in results:
# print r
db.table('tb_pesan_gedung').where('id_pesan', r['id_pesan']).update(status='expired')
# query
# SELECT
# waktu_pesan,
# now() as sekarang,
# DATE_ADD(waktu_pesan, INTERVAL 2 HOUR) as expired,
# if(now() > (DATE_ADD(waktu_pesan, INTERVAL 2 HOUR)), 'lebih','kurang') as jam_setelah
# from tb_pesan_gedung
# where
# (DATE_ADD(NOW(), INTERVAL 2 HOUR) > waktu_pesan)
# and
# status = 'pending'
#install orator
#cara nya: pip install orator
def on_get(self, req, resp):
resp.set_header('Access-Control-Allow-Origin', '*')
resp.set_header('Access-Control-Allow-Methods', 'GET')
resp.set_header('Access-Control-Allow-Headers', 'Content-Type')
# reading config
config = configparser.ConfigParser()
config.readfp(open('config.ini'))
dbhost = config.get('database', 'host')
dbdatabase = config.get('database', 'database')
dbuser = config.get('database', 'user')
dbpassword = config.get('database', 'password')
prefix = config.get('database', 'prefix')
wkhtmltopdf_path = config.get('wkhtmltopdf', 'path')
baseurl = config.get('frontend', 'baseurl')
#Setup DB
config = {
'mysql': {
'driver': 'mysql',
'host': dbhost,
'database': dbdatabase,
'user': dbuser,
'password': dbpassword,
'prefix': ''
}
}
db = DatabaseManager(config)
Model.set_connection_resolver(db)
try:
key = req.params['key']
data = {"status": "success", "data": {}}
except:
resp.status = falcon.HTTP_404
return
try:
val = db.table(prefix + 'files').where('file_key', key).count()
except:
esp.status = falcon.HTTP_503
return
if val > 0:
try:
resp.body = json.dumps({
"status": "success",
"data": {
"file_status": "exists",
"url": baseurl + "file?key=" + key
}
})
resp.status = falcon.HTTP_200
except:
resp.status = falcon.HTTP_404
else:
resp.body = json.dumps({
"status": "success",
"data": {
"file_status": "not exists"
}
})
class BatchWorker(Process):
def __init__(self,task_id,worker_control):
super(BatchWorker, self).__init__()
self.task_id = task_id
self.worker_control = worker_control
self.db = DatabaseManager(DATABASES)
def run(self):
print "-- BATCH WORKER ONLINE : %s" % self.task_id
image_detector = ImageDetector()
tasks = self.db.table("tasks").where("task_id","=",self.task_id).order_by("id","asc").get()
counter = 0
is_ok_to_continue = True
for t in tasks:
counter = counter + 1
try:
if is_ok_to_continue:
target_url = t.target_url
image_url = t.image_url
screen_width = t.screen_width
screen_height = t.screen_height
task_id = "%s_%s" % (self.task_id,counter)
status, image_match = image_detector.process(task_id,target_url,image_url,screen_width,screen_height)
print status
print image_match
self.db.table("tasks").where("task_id","=",t.task_id).update({
"image_path" : image_match,
"status": status,
"processing_status": 2
})
if status:
is_ok_to_continue = False
else:
# at least one of the previous task has succeeded
# the rest of the task will be set to DONE
self.db.table("tasks").where("task_id","=",t.task_id).update({
"image_path" : "",
"processing_status": 2
})
except:
self.db.table("tasks").where("task_id","=",t.task_id).update({
"image_path" : "",
"processing_status": 3
})
self.worker_control.release()
class Classifier:
def __init__(self, train_test_split):
self.split = train_test_split
# read config file
required_fields = {"database": ["host", "database", "user", "password", "idfield", "textfield"]}
self.config = config = configparser.ConfigParser()
config.read('../settings_private.ini')
for key, values in required_fields.items():
for value in values:
if not config.has_option(key, value):
raise Exception("Missing '"+str(value)+"' in section '"+str(key)+"' in settings.ini")
def train(self):
self.pos = pandas.read_csv("../tweets_pos.csv")
self.pos = self.pos["Tweet"].to_list()
self.pos = list(set(self.pos))
self.neg = pandas.read_csv("../tweets_neg.csv")
self.neg = self.neg["Tweet"].to_list()
self.neg = list(set(self.neg))
X_train, X_test = self.buildFeatures()
self.classifier = NaiveBayesClassifier.train(X_train)
print("Accuracy of the classifier: ", nltk.classify.util.accuracy(self.classifier, X_test))
def extract_features(self, word_list):
return dict([(word, True) for word in word_list])
def buildFeatures(self):
pos_th = int(self.split*len(self.pos))
neg_th = int(self.split*len(self.pos))
feat_pos = [(self.extract_features(t.split()), 'Positive') for t in self.pos]
feat_neg = [(self.extract_features(t.split()), 'Negative') for t in self.neg]
return feat_pos[0:pos_th]+feat_neg[0:neg_th], feat_pos[pos_th:]+feat_neg[neg_th:]
def predict(self, text):
print("PREDICT: ", text)
probdist = self.classifier.prob_classify(self.extract_features(str(text).split()))
pred_sentiment = probdist.max()
print("Predicted sentiment: ", pred_sentiment)
print("Probability: ", round(probdist.prob(pred_sentiment), 2))
return pred_sentiment, round(probdist.prob(pred_sentiment), 2)
def mostImportant(self):
print("Top ten most informative words: ")
for item in self.classifier.most_informative_features()[:10]:
print(item[0])
def save(self, filename):
import pickle
f = open(filename, 'wb')
pickle.dump(self.classifier, f)
f.close()
def load(self, filename):
import pickle
f = open(filename, 'rb')
self.classifier = pickle.load(f)
f.close()
# read tweets from a database
def getTweets(self):
config = {
'mysql': {
'driver': 'mysql',
'host': self.config['database']['host'],
'database': self.config['database']['database'],
'user': self.config['database']['user'],
'password': self.config['database']['password'],
'prefix': ''
}
}
self.db = DatabaseManager(config)
tweets = self.db.table(self.config['database']['table']).where(self.config['database']['retweetfield'], '=', '0').lists(self.config['database']['textfield'])
return tweets
def predictFromDb(self):
tweets = self.getTweets()
tweets = [re.sub('<[^<]+?>', '', text) for text in tweets if text[0:3] != "RT "]
preds = []
for tweet in tweets:
pred, acc = self.predict(tweet.replace('\n', ' ').replace('\r', '').replace('"', ""))
if float(acc) < 0.65:
preds.append("Unsure")
print("Unsure: ", acc)
else:
preds.append(pred)
return preds
class Driver(Model):
__table__ = 'drivers'
pass
print("Buscando motoristas...")
contacts = wppDriver.get_contacts()
print(str(len(contacts)) + " Contatos")
drivers = Driver.where_raw('options_stage = 0').get()
phones_that_received = map(lambda x: x.phone, drivers)
new_contacts = [a for a in contacts if a.id not in phones_that_received]
print(str(len(new_contacts)) + "Novos contatos esperando mensagens")
print(datetime.datetime.now())
iteraction = 0
for driver in new_contacts:
print("iteração: "+ str(iteraction))
iteraction += 1
if iteraction == 1000:
print("chegou nos mil")
raise "error"
try:
db.table('drivers').insert(phone=driver.id, name=driver.name, created_at=datetime.datetime.now(), updated_at=datetime.datetime.now())
except:
print("erro no usuário "+ driver.id)
print(datetime.datetime.now())
def on_post(self, req, resp):
resp.set_header('Access-Control-Allow-Origin', '*')
resp.set_header('Access-Control-Allow-Methods', 'GET')
resp.set_header('Access-Control-Allow-Headers', 'Content-Type')
# reading config
config = configparser.ConfigParser()
config.readfp(open('config.ini'))
dbhost = config.get('database', 'host')
dbdatabase = config.get('database', 'database')
dbuser = config.get('database', 'user')
dbpassword = config.get('database', 'password')
prefix = config.get('database', 'prefix')
wkhtmltopdf_path = config.get('wkhtmltopdf', 'path')
baseurl = config.get('frontend', 'baseurl')
#Setup DB
config = {
'mysql': {
'driver': 'mysql',
'host': dbhost,
'database': dbdatabase,
'user': dbuser,
'password': dbpassword,
'prefix': ''
}
}
#TODO create tables
db = DatabaseManager(config)
Model.set_connection_resolver(db)
data = {"status": "failure", "data": {}}
try:
title = req.params['title']
data = {"status": "success", "data": {}}
except:
title = ''
try:
md = req.params['markdown']
data = {"status": "success", "data": {}}
except:
data = {"status": "failure", "data": {}}
md = False
if md != False:
md = head + convertMarkdownToHtml(md.replace("/n", "<br>"))
#md += "<br><div class='end'>www.piratenpartei.de</div>"
key = datetime.datetime.now().strftime(
"%Y-%m-%d_") + randomString()
font_config = FontConfiguration()
file_ = HTML(string=md).write_pdf(font_config=font_config)
try:
db.table(prefix + 'files').insert({
'file_key': key,
'file': file_
})
data = {
"status": "success",
"data": {
"key": key,
"url": baseurl + key
}
}
except Exception as e:
print(e)
data = {"status": "failure", "data": {}}
resp.status = falcon.HTTP_503
return
else:
data = {"status": "failure", "data": {}}
resp.body = json.dumps(data)
if __name__ == '__main__':
print "PROCESSER RUNNING"
print "#CPU COUNT : %s" % psutil.cpu_count()
worker_control = Semaphore(psutil.cpu_count())
while True:
if worker_control.acquire(False):
db.reconnect()
t = db.table("tasks").where("processing_status","=",0).first()
if t is not None:
task_id = t.task_id
db.table("tasks").where("task_id","like",task_id).update({'processing_status': 1})
num_tasks = db.table("tasks").where("task_id","like",task_id).count()
print num_tasks
if num_tasks > 1:
w = BatchWorker(task_id,worker_control)
w.start()
else:
w = SimpleWorker(task_id,worker_control)
parser = argparse.ArgumentParser(description='Update new database from old')
parser.add_argument('database', help='Database file')
args = vars(parser.parse_args())
# Connects to a sqlite database used to store comment ids.
print('Connecting to database...')
try:
connection = sqlite3.connect(args['database'])
connection.row_factory = sqlite3.Row
cursor = connection.cursor()
print('Connected to database!')
except:
print('Connection to database failed.',sys.exc_info()[0])
exit()
cursor.execute('select id,utc_time from comments where utc_time > ' + str(time.time() - 31*24*60*60))
number = 0
inserts = list()
print("Creating list")
for cid in cursor:
if (number > 0) and (number % 250) == 0:
print("Bulk insertion %d" % (number))
db.table('comments').insert(inserts)
inserts = list()
inserts.append({'commentId': cid[0], 'utcTime': cid[1]})
number += 1
print("Bulk insertion")
db.table('comments').insert(inserts)
from orator import DatabaseManager
import logging
logging.basicConfig(level=logging.DEBUG)
# https://orator-orm.com/docs/0.9/query_builder.html
# https://orator-orm.com/docs/0.9/basic_usage.html#query-logging
config = {
"sqlite3": {"driver": "sqlite", "database": "examples.db", "log_queries": True}
}
db = DatabaseManager(config)
with db.transaction():
user = db.table("users").first()
print(user)
with db.transaction():
user = db.table("users").select("id", "name").first()
print(user)
print("----------------------------------------")
for users in db.table("users").select("id", "name").chunk(100):
for user in users:
print(user)
def RegistroLibros(self):
self.__log.info("Ingresando al Registro de Libros")
opcionesRegisLibros = {
"\t- Registrar Libro": 1,
"\t- Listar Libros": 2,
"\t- Eliminar Libros": 3
}
MenuRegisLibros = Menu("Menu Modulo Libros", opcionesRegisLibros)
regisLibros = MenuRegisLibros.mostrarMenu()
menuRegisLib = True
while menuRegisLib:
if (regisLibros == 1):
nuevoLibro = Libro()
nombreLibro = input("escriba el nombre del Libro \n")
isbnLibro = input("escriba el ISBN del libro \n")
autores = Autor()
print(f"\t ID\t Nombre\t Correo")
for obj in autores.all():
print(f"\t {obj.id}\t {obj.nombre}\t {obj.correo}")
print("Escriba el id del Autor")
autor_idLibro = input()
editoriales = Editorial()
print(f"\t ID\t Nombre")
for obj in editoriales.all():
print(f"\t {obj.id}\t {obj.nombre}")
print("Escriba el id de la editorial")
editorialLibro = input()
nuevoLibro.nombre = nombreLibro
nuevoLibro.isbn = isbnLibro
nuevoLibro.autor_id = autor_idLibro
nuevoLibro.estado_libro_id = editorialLibro
nuevoLibro.save()
menuRegisLib = False
elif (regisLibros == 2):
libros = Libro()
print(f"ID\t NOMBRE\t ISBN\t AUTOR\t EDITORIAL\t ESTADO")
for obj in libros.all():
print(
f"\t {obj.id}\t {obj.nombre}\t {obj.ISBN}\t {obj.autors_id}\t {obj.editorial_id}\t {obj.estado_libro_id}"
)
print("Escriba el id de la editorial")
editorialLibro = input()
input("Regresar???")
menuRegisLib = False
program.utils.Menu("Menu Modulo Libros", regisLibros)
elif (regisLibros == 3):
libros = Libro()
print(f"ID\t NOMBRE\t ISBN\t AUTOR\t EDITORIAL\t ESTADO")
for obj in libros.all():
print(
f"\t {obj.id}\t {obj.nombre}\t {obj.isbn}\t {obj.autors_id}\t {obj.editorial_id}\t {obj.estado_libro_id}"
)
print("Escriba el id del libro que se desea eliminar: \n")
id = input()
DatabaseManager.table('libros').get()
DatabaseManager.table('libros').where('id', '=',
f'{id}').delete()
menuRegisLib = False
elif (regisLibros == 9):
__log = log("Saliendo")
break
return str(o)
def dumps(body):
return json.dumps(body or {}, default=converter)
def new_error(message, code):
return {
'code': code,
'message': message,
}
JWT_SECRET = db.table('secret').where('key', 'JWT').pluck('body')
def jwt_encode(user_id):
encoded = jwt.encode({'user_id': user_id}, JWT_SECRET, algorithm='HS256')
return encoded.decode('utf8')
def jwt_decode(raw):
token = raw
if raw.startswith('Bearer'):
token = raw[7:]
try:
with urllib.request.urlopen(api_endpoint) as url:
json_data = json.loads(url.read().decode())
# Set field sizes - we know some are longer than 255
print(f"Setting field sizes")
for field in data_fields.keys():
# Max of length of values of entries in the array
seq = [len(x[field]) for x in json_data if x[field]]
if max(seq) > 255:
data_fields[field]["size"] = max(seq)
else:
data_fields[field]["size"] = 255
# Create a table
print("Creating DB table named 'data'")
with schema.create('data') as table:
for field in data_fields.keys():
if data_fields[field]["type"] == "string": table.string(field, data_fields[field]["size"]).nullable()
if data_fields[field]["type"] == "integer": table.integer(field).nullable()
if data_fields[field]["type"] == "datetime": table.string(field).nullable()
# Fire in the data
print("Writing records")
for record in json_data:
db.table('data').insert(record)
print("Complete!")
comment.reply(commandResponse)
girmRequestsToFind = re.findall(r'\[\s*girm\s*([\d\-,\s]+)\s*\\?\](?im)(?!\()', comment.body)
if girmRequestsToFind:
requestIsValid,commandResponse = girmResponse.getResponse(girmRequestsToFind)
if requestIsValid:
try:
comment.reply(commandResponse)
except praw.exceptions.APIException as e:
if e.error_type == "RATELIMIT":
logger.warning("Sleeping 90 seconds due to RateLimitExceed ")
sleep(90)
comment.reply(commandResponse)
try:
# db.connection().enable_query_log()
logger.info("Cleaning database of old comments")
db.table('comments').where('utcTime', '<', time.time() - 31*24*60*60).delete()
dbComment = Comment()
dbComment.commentId = comment.id
dbComment.utcTime = time.time()
dbComment.save()
logger.info("Inserting comment %s by user %s into database" % (comment.id, comment.author.name))
except:
logger.critical("Database insert for comment %s by user %s failed." % (comment.id, comment.author.name),sys.exc_info()[0],"x")
exit()
except requests.exceptions.HTTPError:
logger.error("HTTP Error: waiting 5 minutes to retry"+sys.exc_info()[0],"e")
sleep(5*60)
