def test_database():
test_db = SqliteExtDatabase(":memory:")
with test_db.bind_ctx(ALL_MODELS):
test_db.create_tables(ALL_MODELS)
try:
yield
finally:
test_db.drop_tables(ALL_MODELS)
test_db.close()
def memdb():
mdb = SqliteExtDatabase(':memory:')
models = [db.Job]
mdb.bind(models)
mdb.connect()
mdb.create_tables(models)
yield mdb
mdb.close()
def init_database(self):
migrate_db = SqliteExtDatabase(self.config.database.path)
# Run migrations
del (logging.getLogger('peewee_migrate').handlers[:])
router = Router(migrate_db)
router.run()
migrate_db.close()
self.db = SqliteQueueDatabase(self.config.database.path)
models = [Event]
self.db.bind(models)
def import_song_statistic_from(self, database_file):
self.logger.info('Importing database songs statistics')
try:
from playhouse.sqlite_ext import SqliteExtDatabase
other_database = SqliteExtDatabase(database_file)
cursor = other_database.execute_sql(
"SELECT ADDED, LAST_PLAYED, PLAYED, TITLE, ARTIST, ALBUM FROM SONG"
)
with database_context.atomic():
for row in cursor.fetchall():
database_context.execute_sql(
"UPDATE SONG SET ADDED = ?, LAST_PLAYED = ?, PLAYED = ? WHERE TITLE = ? AND ARTIST = ? AND ALBUM = ?",
row)
other_database.close()
except:
self.logger.exception(
'Error while importing database songs statistics')
def init_database(self):
# Migrate DB location
old_db_path = os.path.join(CLIPS_DIR, "frigate.db")
if not os.path.isfile(
self.config.database.path) and os.path.isfile(old_db_path):
os.rename(old_db_path, self.config.database.path)
# Migrate DB schema
migrate_db = SqliteExtDatabase(self.config.database.path)
# Run migrations
del logging.getLogger("peewee_migrate").handlers[:]
router = Router(migrate_db)
router.run()
migrate_db.close()
self.db = SqliteQueueDatabase(self.config.database.path)
models = [Event, Recordings]
self.db.bind(models)
body=resp)
break
n -= 1
except Exception as e:
print("[E] Error :", e)
time.sleep(0.1)
mybank = User.get(npm=user_id)
total += mybank.saldo
print("finish poll")
print("total :", total, " sender :", sender_id)
resp['nilai_saldo'] = total
resp = json.dumps(resp)
return channel.basic_publish(exchange='EX_GET_TOTAL_SALDO',
routing_key='RESP_' + sender_id,
body=resp)
def request_get_total_saldo(ch, method, properties, body):
try:
msg = json.loads(body.decode("utf-8"))
print("[x] ", msg['action'], " message :", msg)
get_total_saldo(msg)
except Exception as e:
print("[E] Error :", e)
channel.basic_consume(request_get_total_saldo, queue=queue_name, no_ack=True)
channel.start_consuming()
database.close()
class BefrankPipeline(object):
def __init__(self):
self.db = None
self.setup_db_connection()
self.create_tables()
def setup_db_connection(self):
self.db = SqliteExtDatabase('movies.db')
self.db.connect()
def create_tables(self):
return self.db.create_tables[Movie, Star, Writer, Director, Genre,
PlotKeyword]
def close_db(self):
self.db.close()
def process_item(self, item, spider):
for key, value in six.iteritems(item):
if key == "CastMembers":
continue
if isinstance(value, list):
if value:
templist = []
for obj in value:
temp = self.stripHTML(obj)
templist.append(temp)
item[key] = templist
else:
item[key] = ""
elif key is not 'MainPageUrl':
item[key] = self.stripHTML(value)
else:
item[key] = value
self.store_in_db(item)
return item
def store_in_db(self, item):
self.storeFilmInfoInDb(item)
film_id = self.cursor.lastrowid
for cast in item['CastMembers']:
self.storeActorInfoInDb(cast, film_id)
def storeFilmInfoInDb(self, item):
self.cur.execute("INSERT INTO Films(\
title, \
rating, \
ranking, \
release_date, \
page_url, \
director, \
writers, \
runtime, \
sinopsis, \
genres, \
mpaa_rating, \
budget, \
language, \
country, \
gross_profit, \
opening_weekend_profit, \
aspect_ratio, \
sound_mix, \
color\
) \
VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? )" , \
( \
item.get('Title', ''),
float(item.get('Rating', 0.0)),
int(item.get('Ranking', 0)),
item.get('ReleaseDate', ''),
item.get('MainPageUrl', ''),
', '.join(item.get('Director', '')),
', '.join(item.get('Writers', '')),
item.get('Runtime', ''),
item.get('Sinopsis', '').strip(),
', '.join(item.get('Genres', '')),
item.get('MpaaRating', ''),
self.cleanMoney(item.get('Budget', '')),
item.get('Language', ''),
item.get('Country', ''),
self.cleanMoney(item.get('GrossProfit', '')),
self.cleanMoney(item.get('OpeningWeekendProfit', '')),
item.get('AspectRatio', '').strip(),
', '.join(item.get('SoundMix', '')),
item.get('Color', '')
))
self.con.commit()
def storeActorInfoInDb(self, item, film_id):
self.cur.execute(
"INSERT INTO Actors(\
film_id, \
actor_name, \
charecter_name, \
ranking \
) \
VALUES(?,?,?,?)",
(film_id, self.stripHTML(item.get('ActorName', '')).strip(),
self.stripHTML(item.get('CharacterName',
'')).strip(), item.get('Ranking', 0)))
self.con.commit()
def stripHTML(self, string):
tagStripper = MLStripper()
tagStripper.feed(string)
return tagStripper.get_data()
def cleanMoney(self, string):
# you could add more simpbles to this, but it gets kinda complex with some of the symbles being unicode, so I skpped that for now.
currencySymbles = "$"
cleanMoneyString = ""
stopAdding = False
for index, char in enumerate(list(string)):
if char in currencySymbles and not stopAdding:
cleanMoneyString += char
elif char == "," and not stopAdding:
cleanMoneyString += char
elif char.isdigit() and not stopAdding:
cleanMoneyString += char
elif char in ' ':
# we know that numbers do not have spaces in them, so we can assume that once the number
# has started there will be no spaces
if len(cleanMoneyString) > 0:
stopAdding = True
return cleanMoneyString
def libgen(mol_list, output_name):
"""
function to generate a database format library of fragments from a mol, list of mol objects, .smi, or .sdf file
:param mol_list: list of molecules, a single molecule, or a filename of molecules to read
:type mol_list: str|Chem.Mol|[Chem.Mol]
:param output_name: name of the database to use?
:type output_name: str
:return:
"""
# if a file not a list then read into list
if isinstance(mol_list, str) and mol_list.endswith(".smi"):
mol_list = Chem.SmilesMolSupplier(mol_list,
delimiter="\t",
titleLine=False)
elif isinstance(mol_list, str) and mol_list.endswith(".sdf"):
mol_list = Chem.SDMolSupplier(mol_list)
elif type(mol_list) == Chem.Mol:
mol_list = [mol_list]
elif type(mol_list) == list:
assert type(mol_list[0]) == Chem.Mol
else:
raise Exception(
"Did you provide a list of mol objects? Input type error.")
fragment_dict_deque = deque()
heritage_dict_deque = deque()
atoms_dict_deque = deque()
pseudoatoms_dict_deque = deque()
logger.info("Fragmenting:")
n = len(mol_list)
i = 0
t0 = time.time()
for mol in mol_list:
re_mol = RecomposerMol.fromMol(mol=mol)
frag_list, heritage_list, atoms_list, pseudo_atoms_list = re_mol.get_all_fragments(
7)
fragment_dict_deque.extend(frag_list)
heritage_dict_deque.extend(heritage_list)
atoms_dict_deque.extend(atoms_list)
pseudoatoms_dict_deque.extend(pseudo_atoms_list)
logger.info("DONE: %d/%d %.f" % (i, n, 1000 * (time.time() - t0) /
(i + 1)))
i += 1
logger.info("Done")
logger.info("Saving:")
# create the database for the output
db = SqliteExtDatabase(
output_name,
pragmas={
'cache_size': -1024 * 64, # 64MB page-cache.
'journal_mode':
'wal', # Use WAL-mode (you should always use this!).
'foreign_keys': 0,
'wal_autocheckpoint': 10,
})
db.connect()
# get the models
Fragment, Heritage, PseudoAtoms, Atoms = lib_read(db)
Fragment.create_table(safe=True)
Heritage.create_table(safe=True)
PseudoAtoms.create_table(safe=True)
Atoms.create_table(safe=True)
with db.atomic():
if len(fragment_dict_deque) > 0:
for ents in chunked(fragment_dict_deque, 200):
query = Fragment.replace_many(ents)
query.execute()
for ents in chunked(heritage_dict_deque, 200):
query = Heritage.replace_many(ents)
query.execute()
for ents in chunked(pseudoatoms_dict_deque, 200):
query = PseudoAtoms.replace_many(ents)
query.execute()
for ents in chunked(atoms_dict_deque, 200):
query = Atoms.replace_many(ents)
query.execute()
db.close()
clean(output_name)
return 1
class BaseModel(Model):
class Meta: # noqa
database = db
class Video(BaseModel):
id = TextField(primary_key=True)
url = TextField()
title = TextField()
db.connect()
Video.create_table(True)
db.close()
def is_admin(user):
if user['hash'] == config['owner']:
logging.info("%s(%s) authenticated as owner", user['name'],
user['hash'])
return 2
if user['hash'] in config.as_list('admins'):
logging.info("%s(%s) authenticated as admin", user['name'],
user['hash'])
return 1
logging.info("Failed to authenticate %s(%s)", user['name'], user['hash'])
return 0
class Channel(BaseModel):
name = CharField(unique=True)
num = IntegerField(default=0)
class ChannelPublisher(BaseModel):
ch_id = ForeignKeyField(Channel, related_name='channel_pub')
pub_id = ForeignKeyField(User, related_name='publisher')
class ChannelSubscriber(BaseModel):
ch_id = ForeignKeyField(Channel, related_name='channel_sub')
sub_id = ForeignKeyField(User, related_name='subscriber')
ts = IntegerField()
class Feed(BaseModel):
ch_id = ForeignKeyField(Channel, related_name='channel_feed')
pub_id = ForeignKeyField(User, related_name='publisher_feed')
text = CharField()
sid = IntegerField()
db.connect()
print 'Database Connected'
if __name__ == "__main__":
if len(sys.argv) >=2 and sys.argv[1] == 'clear':
db.close()
os.remove('my_database.db')
db.connect()
db.create_tables([User, Channel, ChannelPublisher, ChannelSubscriber, Feed])
db.close()
print 'Database cleared'
