def download(name,force=False):
db=TinyDB(path_db_)
temp = Query()
data=requests.get("https://raw.githubusercontent.com/PyThaiNLP/pythainlp-corpus/master/db.json")
data_json=data.json()
if name in list(data_json.keys()):
temp_name=data_json[name]
print("Download : "+name)
if len(db.search(temp.name==name))==0:
print(name+" "+temp_name['version'])
download_(temp_name['download'],temp_name['file_name'])
db.insert({'name': name, 'version': temp_name['version'],'file':temp_name['file_name']})
else:
if len(db.search(temp.name==name and temp.version==temp_name['version']))==0:
print("have update")
print("from "+name+" "+db.search(temp.name==name)[0]['version']+" update to "+name+" "+temp_name['version'])
yes_no="y"
if force==False:
yes_no=str(input("y or n : ")).lower()
if "y"==yes_no:
download_(temp_name['download'],temp_name['file_name'])
db.update({'version':temp_name['version']},temp.name==name)
else:
print("re-download")
print("from "+name+" "+db.search(temp.name==name)[0]['version']+" update to "+name+" "+temp_name['version'])
yes_no="y"
if force==False:
yes_no=str(input("y or n : ")).lower()
if "y"==yes_no:
download_(temp_name['download'],temp_name['file_name'])
db.update({'version':temp_name['version']},temp.name==name)
db.close()
def test_write_first_cache(self):
"""test write behavior on first pass (cache-buster mode)"""
self.test_clear_existing_cache() #blowup existing cache again
dummy_data = forecast_utils.parse_emd_data(DEMO_DATA['result'])
forecast_utils.write_prediction_cache(
self.region_id,
self.type_id,
dummy_data,
cache_path=self.cache_path
)
assert path.isfile(self.cache_filepath)
tdb = TinyDB(self.cache_filepath)
data = tdb.all()[0]
keys_list = [
'cache_date',
'region_id',
'type_id',
'lastWrite',
'prediction'
]
assert set(keys_list) == set(data.keys())
dummy_str_data = dummy_data.to_json(
date_format='iso',
orient='records'
)
cached_data = pd.read_json(data['prediction'])
assert data['prediction'] == dummy_str_data
tdb.close()
def check_prediction_cache(
region_id,
type_id,
cache_path=CACHE_PATH,
db_filename='prophet.json'
):
"""check tinyDB for cached predictions
Args:
region_id (int): EVE Online region ID
type_id (int): EVE Online type ID
cache_path (str): path to caches
db_filename (str): name of tinydb
Returns:
pandas.DataFrame: cached prediction
"""
utc_today = datetime.utcnow().strftime('%Y-%m-%d')
prediction_db = TinyDB(path.join(cache_path, db_filename))
raw_data = prediction_db.search(
(Query().cache_date == utc_today) &
(Query().region_id == region_id) &
(Query().type_id == type_id)
)
prediction_db.close()
if raw_data:
panda_data = pd.read_json(raw_data[0]['prediction'])
return panda_data
else:
return None
def index():
form = SearchForm()
query = request.args.get('query', '').strip()
db = TinyDB(recipyGui.config.get('tinydb'))
if not query:
runs = db.all()
else:
# Search run outputs using the query string
runs = db.search(
where('outputs').any(lambda x: listsearch(query, x)) |
where('inputs').any(lambda x: listsearch(query, x)) |
where('script').search(query) |
where('notes').search(query) |
where('unique_id').search(query))
runs = sorted(runs, key = lambda x: parse(x['date'].replace('{TinyDate}:', '')) if x['date'] is not None else x['eid'], reverse=True)
run_ids = []
for run in runs:
if 'notes' in run.keys():
run['notes'] = str(escape(run['notes']))
run_ids.append(run.eid)
db.close()
return render_template('list.html', runs=runs, query=query, form=form,
run_ids=str(run_ids),
dbfile=recipyGui.config.get('tinydb'))
def write(message, args):
# message.reply('I can understand hi or HI!')
# react with thumb up emoji
#message.react('+1')
db = TinyDB('db.json')
db.insert({'value': args});
print args
db.close()
def log_output(filename, source):
filename = os.path.abspath(filename)
if option(CONFIG, 'general', 'debug'):
print("Output to %s using %s" % (filename, source))
#Update object in DB
db = TinyDB(DBFILE)
db.update(append("outputs", filename), eids=[RUN_ID])
db.close()
def get_genres(database_name):
"""Utility method to get all the genres as a set"""
db = TinyDB(os.path.join(os.getcwd(), database_name))
all_genres = { song['genre'] for song in db.all() }
specific_genres = set()
for genre in all_genres:
specific_genres = specific_genres.union(set(genre.strip().split('/')))
db.close()
return _strip_spaces(specific_genres)
def get_file(name):
db=TinyDB(path_db_)
temp = Query()
if len(db.search(temp.name==name))>0:
path= get_path_data(db.search(temp.name==name)[0]['file'])
db.close()
if not os.path.exists(path):
download(name)
return path
def api():
db = TinyDB(DB_FILENAME)
results = db.search((Query().hidden == False) | (Query().starred == True))
db.close()
for result in results:
result['eid'] = result.eid
result['url'] = url_for('goto', eid=result.eid)
results.sort(key=lambda r: r['created_at'])
return json.dumps(results)
def runs2json():
run_ids = literal_eval(request.form['run_ids'])
db = TinyDB(recipyGui.config.get('tinydb'))
runs = [db.get(eid=run_id) for run_id in run_ids]
db.close()
response = make_response(dumps(runs, indent=2, sort_keys=True))
response.headers['content-type'] = 'application/json'
response.headers['Content-Disposition'] = 'attachment; filename=runs.json'
return response
def test_gc(tmpdir):
# See https://github.com/msiemens/tinydb/issues/92
path = str(tmpdir.join('db.json'))
db = TinyDB(path)
table = db.table('foo')
table.insert({'something': 'else'})
table.insert({'int': 13})
assert len(table.search(where('int') == 13)) == 1
assert table.all() == [{'something': 'else'}, {'int': 13}]
db.close()
def hide_all():
db = TinyDB(DB_FILENAME)
eids = [
r.eid for r in db.search(
(Query().hidden == False) & (Query().starred == False)
)
]
db.update({'hidden': True}, eids=eids)
db.close()
return 'OK'
def patched_modules():
db = TinyDB(recipyGui.config.get('tinydb'))
modules = db.table('patches').all()
db.close()
form = SearchForm()
return render_template('patched_modules.html', form=form,
active_page='patched_modules', modules=modules,
dbfile=recipyGui.config.get('tinydb'))
class TinyDBConvertor(object):
def __init__(self, collection_name, db_name, db_path=LOCAL_DIR):
self.local_db_fullpath = os.path.join(db_path, db_name)
self.local_db = TinyDB(self.local_db_fullpath)
self.collection_name = collection_name
def reload(self):
self.local_db.close()
self.local_db = TinyDB(self.local_db_fullpath)
def find(self):
self.reload()
query = Query()
resp = self.local_db.search(query.type == self.collection_name)
self.local_db.close()
return resp
def find_one(self, entry):
self.reload()
query = Query()
resp = self.local_db.search((query.type == self.collection_name) &
(query.fqu == entry['fqu']))
self.local_db.close()
if len(resp) == 0:
return None
else:
return resp[0]
def save(self, new_entry):
self.reload()
new_entry['type'] = self.collection_name
resp = self.local_db.insert(new_entry)
self.local_db.close()
return resp
def remove(self, entry):
self.reload()
query = Query()
resp = self.local_db.remove((query.type == self.collection_name) &
(query.fqu == entry['fqu']))
self.local_db.close()
return resp
def annotate():
notes = request.form['notes']
run_id = int(request.form['run_id'])
query = request.args.get('query', '')
db = TinyDB(recipyGui.config.get('tinydb'))
db.update({'notes': notes}, eids=[run_id])
db.close()
return redirect(url_for('run_details', id=run_id, query=query))
def run_details():
form = SearchForm()
annotateRunForm = AnnotateRunForm()
query = request.args.get('query', '')
run_id = int(request.args.get('id'))
db = TinyDB(recipyGui.config.get('tinydb'))
r = db.get(eid=run_id)
db.close()
return render_template('details.html', query=query, form=form,
annotateRunForm=annotateRunForm, run=r)
class FacebookPub:
def __init__(self, appConfig, dbName='fb_db.json') -> None:
self.db = TinyDB(dbName)
self.config = appConfig
self.query = Query()
print("Facebook publisher created.")
self.parent_object = self.config.getProperty(
'Publishers.Facebook.ParentObject')
self.access_token = self.config.getProperty(
'Publishers.Facebook.AccessToken')
def __del__(self):
self.db.close()
def FormatMessage(self, rawMessages):
res = 'Automatsko obaveštenje o nestanku struje u selu i okolini:\n\n'
for rawMessage in rawMessages:
res = res + rawMessage['common_desc'] + '\n'
for rawMessageDesc in rawMessage['desc']:
pretty = ' '.join(rawMessageDesc.split('\\n'))
pretty = ' '.join(pretty.split('\\t'))
pretty = ' '.join(pretty.split('\n'))
pretty = ' '.join(pretty.split('\t'))
pretty = ' '.join(pretty.split(' '))
res = res + pretty + '\n'
res = res + '\n'
return res
def Publish(self, message) -> None:
if len(message.message) > 0:
if len(self.db.search(self.query.hash == message.hash)) > 0:
print('Already published to Facebook')
else:
try:
print('Posting to FB...' +
self.FormatMessage(message.message))
post = {
'message': self.FormatMessage(message.message),
'link': message.link
}
graph = facebook.GraphAPI(access_token=self.access_token,
version='3.1')
api_request = graph.put_object(
parent_object=self.parent_object,
connection_name='feed',
message=post['message'],
#link=post['link']
)
except:
print('Posting to FB failed')
self.db.insert(message.ToDict())
def updateCodalListOfSoratMali(chrome, symbol, codalRawDataTableName, sheetId):
## TrackNo_ValueNumber
db = TinyDB(
FilenameManager.get({
'enum': FilenameManager.CodalLinks,
'symbol': symbol
}))
LinkOfPagesTable = db.table('Links')
LinkOfPages = LinkOfPagesTable.all()
theLetterInLinkOfPages = []
for linkOfPage in LinkOfPages:
title = linkOfPage['letter']['Title']
if (title.find("صورتهای مالی") > -1):
theLetterInLinkOfPages.append(linkOfPage)
codalRawDataDB = TinyDB(
FilenameManager.get({
'enum': FilenameManager.CodalRawData,
'symbol': symbol
}))
theSubjectsTable = codalRawDataDB.table(codalRawDataTableName)
theSubjectsTableData = theSubjectsTable.all()
theSubjectsTableDataLen = len(theSubjectsTableData)
linkOfPageCounter = 0
for linkOfPage in theLetterInLinkOfPages:
if (len(theLetterInLinkOfPages) <= theSubjectsTableDataLen):
break
isFind = False
for theSubjectRecord in theSubjectsTable:
if (theSubjectRecord['TracingNo'] == linkOfPage['letter']
['TracingNo']):
isFind = True
break
if not (isFind):
url = "https://www.codal.ir" + linkOfPage['letter'][
'Url'] + "&sheetId=" + str(sheetId)
print(str(linkOfPageCounter) + "/" +
str(len(theLetterInLinkOfPages)),
end="\r",
flush=True)
saveListOfSoratHayeMaliToJson(theSubjectsTable, chrome, url, linkOfPage['letter']['Title'], linkOfPage['letter']['TracingNo']\
, linkOfPage['letter']['Symbol'], linkOfPage['letter']['CompanyName'], linkOfPage['letter']['SentDateTime'])
theSubjectsTableDataLen += 1
linkOfPageCounter += 1
codalRawDataDB.close()
def parse_form(form_data, kind="list"):
db = TinyDB(cfg.config['db_name'])
Record = Query()
print("incoming form data:")
# print(form_data)
# print("converted to a dict")
print(dict(form_data))
# form data will have keys, values that may be lists or a single string.
form_data_dict = dict(form_data)
if kind == 'list':
# form data is coming from the 'lineup' page,
# which can have multiple changes on multiple assets
for k, v in form_data_dict.items():
if k != "action":
if isinstance(v, list):
# it's a list of strings.
for item in v:
# check if empty string
if item:
asset_id, field, new_value = item.split('__')
print(
f"++++++++\nSetting this item: {asset_id} to {field}: {new_value}\n++++++++"
)
db.update({field: int(new_value)},
Record.asset_id == asset_id)
else:
# check if empty string
if v:
asset_id, field, new_value = v.split('__')
print(
f"++++++++\nSetting this item: {asset_id} to {field}: {new_value}\n++++++++"
)
db.update({field: int(new_value)},
Record.asset_id == asset_id)
else:
# form data is coming from the 'item' page instead,
# mutiple changes possible but only 1 asset affected
post_update = {}
asset_id = form_data_dict['asset_id'][0]
for x in ['draft_user', 'rank', 'rank_time']:
if form_data_dict[x][0] != '':
post_update[x] = int(form_data_dict[x][0])
for x in ['label_user', 'title_user', 'desc_user']:
if form_data_dict[x][0] != '':
post_update[x] = smartypants.smartypants(
form_data_dict[x][0].strip())
print("Data to update:")
print(post_update)
db.update(post_update, Record.asset_id == asset_id)
db.close()
return
class TelegramPub:
def __init__(self, appConfig, dbName='tg_db.json') -> None:
self.config = appConfig
self.bot = Bot(
token=self.config.getProperty('Publishers.Telegram.Token'))
self.chat_id = self.config.getProperty('Publishers.Telegram.ChatId')
self.db = TinyDB(dbName)
self.query = Query()
print("Telegram publisher created.")
def __del__(self):
self.db.close()
def FormatMessage(self, rawMessages):
res = '*Automatsko obaveštenje o najavljenim prekidima snabdevanja električnom energijom u Banatskom Velikom Selu i okolini*\n\n'
for rawMessage in rawMessages:
common_desc = rawMessage['common_desc']
common_desc = ' '.join(common_desc.split('\\t'))
common_desc = ' '.join(common_desc.split('\n'))
common_desc = ' '.join(common_desc.split('\t'))
common_desc = ' '.join(common_desc.split())
common_desc = common_desc.replace('\xa0', ' ').replace('.', r'\.')
res = res + common_desc + '\n'
for rawMessageDesc in rawMessage['desc']:
pretty = ' '.join(rawMessageDesc.split('\\n')).replace(
'\xa0', ' ').replace('.', '\.')
pretty = ' '.join(pretty.split('\\t'))
pretty = ' '.join(pretty.split('\n'))
pretty = ' '.join(pretty.split('\t'))
pretty = ' '.join(pretty.split())
pretty = re.sub(r'\b[0-9]\\\.', '', pretty)
pretty = re.sub(r'[0-9]\\\.', '', pretty)
res = res + '_' + pretty + '_ \n'
res = res + '[Elektrodistribucija](' + rawMessage['url'] + ')'
res = res + '\n\n'
return res
def Publish(self, message) -> None:
if len(message.message) > 0:
if len(self.db.search(self.query.hash == message.hash)) > 0:
print('Already published to Telegram')
else:
try:
print('Posting to Telegram...' +
self.FormatMessage(message.message))
self.bot.sendMessage(self.chat_id,
self.FormatMessage(message.message),
parse_mode=ParseMode.MARKDOWN_V2)
self.db.insert(message.ToDict())
except:
print('Posting to Telegram failed')
class Library_DB:
DATABASE_FILE = 'db.json'
def __init__(self):
self.db = TinyDB(self.DATABASE_FILE)
def insert_patron(self, patron):
if not patron:
return None
if self.retrieve_patron(patron.get_memberID()): # patron already in db
return None
data = self.convert_patron_to_db_format(patron)
id = self.db.insert(data)
return id
def get_patron_count(self):
results = self.db.all()
return len(results)
def get_all_patrons(self):
results = self.db.all()
return results
def update_patron(self, patron):
if not patron:
return None
query = Query()
data = self.convert_patron_to_db_format(patron)
self.db.update(data, query.memberID == patron.get_memberID())
def retrieve_patron(self, memberID):
query = Query()
# assuming no two people in the db have the same memberID
results = self.db.search(query.memberID == memberID)
if results:
return Patron(results[0]['fname'], results[0]['lname'],
results[0]['age'], results[0]['memberID'])
return None
def close_db(self):
self.db.close()
def convert_patron_to_db_format(self, patron):
return {
'fname': patron.get_fname(),
'lname': patron.get_lname(),
'age': patron.get_age(),
'memberID': patron.get_memberID(),
'borrowed_books': patron.get_borrowed_books()
}
def openPerson(phone):
fileLocation = "files/person_" + phone[0] + ".json"
db = TinyDB(fileLocation)
Person = Query()
data = db.get(Person.phone == phone)
if data:
newPersonObject = personObject(data['name'], data['phone'],
data['token'], data['activities'],
data['invites'], data['friendNumbers'],
data['friendNames'], data['groups'], 0)
db.close()
return newPersonObject
else:
return
def DBFieldList(linkDB=None):
if linkDB != None:
db = TinyDB("{}.json".format(linkDB))
else:
db = TinyDB("DB/{}.json".format(DB_NOM))
D = db
L = dict()
E = dict()
for x in D:
for y in x:
E = {y: x[y]}
L.update(E)
db.close()
return L
def OwnerSessionExist(value, linkDB=None):
"""
Vérifie l'existance d'une session créée par Owner
"""
if linkDB != None:
db = TinyDB("{}.json".format(linkDB))
else:
return 404
if db.search(Query().owner == value) == []:
db.close()
return False
else:
db.close()
return True
def latest_run():
form = SearchForm()
annotateRunForm = AnnotateRunForm()
db = TinyDB(recipyGui.config.get('tinydb'))
runs = db.all()
runs = sorted(runs, key = lambda x: parse(x['date'].replace('{TinyDate}:', '')), reverse=True)
db.close()
return render_template('details.html', query='', form=form, run=runs[0],
annotateRunForm=annotateRunForm,
active_page='latest_run')
def set_value(value_list, value):
# we will be getting a list of rank values: record-id_new-rank
# ['8605132_1', '8605133_0', '8605134_0',]
db = TinyDB('db.json')
Record = Query()
for item in value_list:
if item:
asset_id, new_value = item.split('__')
db.update({value: new_value}, Record.asset_id == asset_id)
print(
f"++++++++\nSetting this item: {asset_id} to {value}: {new_value}\n++++++++"
)
db.close()
return
def download(name: str, force: bool = False):
"""
Download corpus
:param string name: corpus name
:param bool force: force install
"""
db = TinyDB(corpus_db_path())
temp = Query()
data = requests.get(corpus_db_url())
data_json = data.json()
if name in list(data_json.keys()):
temp_name = data_json[name]
print("Download:", name)
if not db.search(temp.name == name):
print(name + " " + temp_name["version"])
download_(temp_name["download"], temp_name["file_name"])
db.insert({
"name": name,
"version": temp_name["version"],
"file": temp_name["file_name"],
})
else:
if not db.search(temp.name == name
and temp.version == temp_name["version"]):
print("Alert: New version is ready to be updated.")
print("from " + name + " " +
db.search(temp.name == name)[0]["version"] +
" update to " + name + " " + temp_name["version"])
yes_no = "y"
if not force:
yes_no = str(input("yes or no (y / n) : ")).lower()
if "y" == yes_no:
download_(temp_name["download"], temp_name["file_name"])
db.update({"version": temp_name["version"]},
temp.name == name)
else:
print("Redownload")
print("from " + name + " " +
db.search(temp.name == name)[0]["version"] +
" update to " + name + " " + temp_name["version"])
yes_no = "y"
if not force:
yes_no = str(input("yes or no (y / n) : ")).lower()
if "y" == yes_no:
download_(temp_name["download"], temp_name["file_name"])
db.update({"version": temp_name["version"]},
temp.name == name)
db.close()
def parse_form(form_data, kind="list"):
db = TinyDB('db.json')
Record = Query()
print("incoming form data:")
print(form_data)
# form data will have keys, values that may be lists or a single string.
form_data_dict = dict(form_data)
print("++++\nconverted to a dict")
print(form_data_dict)
for key in form_data.keys():
for value in form_data.getlist(key):
print(key, ":", value)
db.close()
return
def update(self, citizen):
db = TinyDB(self.db_filename)
query = Query()
if citizen.name != None:
if citizen.score > 100:
citizen.score = 100
if citizen.score < 0:
citizen.score = 0
# Updated if exists, inserted if not
db.upsert(citizen.toDict(), query.name == citizen.name)
db.close()
return True
db.close()
return False
def update_tinydb(self, tinyrundb_json, run_json, rerunWhat):
db = TinyDB(tinyrundb_json, sort_keys=True, indent=4, separators=(',', ': '))
db.drop_table('_default')
query = Query()
runcases = json.loads(open(run_json).read())
for case in runcases:
if case['feature'] in db.tables():
feature_table = db.table(case['feature'])
if (rerunWhat is not None) and (len(feature_table.search(query.status == rerunWhat)) > 0):
feature_table.update({'status': 'rerun'})
else:
feature_table = db.table(case['feature'])
feature_table.insert(case)
db.close()
def valueAt(ID, fieldName, linkDB=None):
"""
Permet de récupérer la valeur contenue dans le champ fieldName de ID
ID: int de l'ID du joueur
fieldName: string du nom du champ à chercher
"""
if linkDB != None:
db = TinyDB("{}.json".format(linkDB))
else:
db = TinyDB("DB/{}.json".format(DB_NOM))
value = db.search(Query().ID == ID)[0][fieldName]
db.close()
return value
class Driver:
"""
Holds the driver state so the flasked script can change behaviour based
on what the user injects via HTTP
"""
name = None
base_path = None
storage_path = None
db = None
proxy_host = None
proxy_port = None
def storage(self, storage_path):
self.storage_path = storage_path
def root(self, base_path):
self.base_path = base_path
def start(self, name):
filename = os.path.join(self.base_path, "{}.yaml".format(name))
if not os.path.exists(filename):
return {
"state": "error",
"message": "Ruleset {} not found".format(filename)
}
self.name = name
self.db = TinyDB(
os.path.join(self.storage_path, "{}-store.json".format(name)))
return {"ruleset": self.name, "state": "started"}
def stop(self):
if self.name is None:
return {"ruleset": None, "state": None}
message = {"ruleset": self.name, "state": "stopped"}
self.name = None
self.db.close()
self.db = None
return message
def state(self):
if self.name is None:
return {"ruleset": None, "state": None}
return {"ruleset": self.name, "state": "running"}
def write_prediction_cache(
region_id,
type_id,
prediction_data,
cache_path=CACHE_PATH,
db_filename='prophet.json',
logger=logging.getLogger('publicAPI')
):
"""update tinydb latest prediction
Args:
region_id (int): EVE Online region ID
type_id (int): EVE Online type ID
prediction_data (:obj:`pandas.DataFrame`): data to write to cache
cache_path (str, optional): path to caches
db_filename (str, optional): name of tinydb
Returns:
None
"""
logger.info('--caching result')
utc_today = datetime.utcnow().strftime('%Y-%m-%d')
prediction_db = TinyDB(path.join(cache_path, db_filename))
## clear previous cache ##
prediction_db.remove(
(Query().cache_date <= utc_today) &
(Query().region_id == region_id) &
(Query().type_id == type_id)
)
## Prepare new entry for cache ##
cleaned_data = prediction_data.to_json(
date_format='iso',
orient='records'
)
data = {
'cache_date': utc_today,
'region_id': region_id,
'type_id': type_id,
'lastWrite': datetime.utcnow().timestamp(),
'prediction':cleaned_data
}
logger.debug(data)
prediction_db.insert(data)
prediction_db.close()
def update_article(self):
inputs = self.get_input()
if web.ctx.method == "GET":
article_id = inputs.get("article_id")
category_list = Categories.select().where(Categories.status == 0)
article = Articles.get_or_none(Articles.id == article_id)
print(article.id)
self.private_data["article"] = article
self.private_data["category_list"] = category_list
return self.display("admin/update_article")
else:
article_id = inputs.get("article_id")
name = inputs.get('name')
content = inputs.get('content')
summary = inputs.get("summary")
category_id = inputs.get("category_id")
source_url = inputs.get("source_url", "")
keywords = str(inputs.get("keywords", "")).strip()
article = Articles.get_or_none(Articles.id == article_id)
try:
tags_list = keywords.split("，") if keywords else []
if tags_list:
got_tags = Tags.select().where(Tags.name.in_(tags_list))
tmp_list = []
for tag in got_tags:
tmp_list.append(tag.name)
for tag_str in tags_list:
tag_str.strip()
if tag_str not in tmp_list:
t = Tags(name=tag_str)
t.save()
db = TinyDB('settings/db.json')
db.truncate()
db.close()
article.update(name=name,
content=content,
summary=summary,
category_id=category_id,
original_address=source_url,
keywords=keywords,
updateTime=time()).where(
Articles.id == article_id).execute()
self.private_data["update_success"] = True
return web.seeother(self.make_url('articles'))
except Exception as e:
log.error('update article failed %s' % traceback.format_exc())
log.error('input params %s' % inputs)
self.private_data["update_success"] = False
return web.seeother(self.make_url('update_article'))
def savePerson(self):
fileLocation = "files/person_" + self.phone[0] + ".json"
db = TinyDB(fileLocation)
db.insert({
'name': self.name,
'phone': self.phone,
'activationCode': self.activationCode,
'token': self.notifToken,
'activities': self.activities,
'invites': self.invites,
'friendNumbers': self.friendNumbers,
'friendNames': self.friendNames,
'groups': self.groups
})
db.close()
def thread_existed(t_url):
m = re.search("m\?kz=(\d+)", t_url)
id = int(m.groups(0)[0])
if not os.path.isfile(env.work_env.stat_db_fullname):
return False
db = TinyDB(env.work_env.stat_db_fullname)
table = db.table(__table_database_id_name__)
query = Query()
if table.search(query.t_id == id):
db.close()
return True
else:
store_thread_id(id)
db.close()
return False
def deleteGroup(self, groupName):
fileLocation = "files/person_" + self.phone[0] + ".json"
db = TinyDB(fileLocation)
Person = Query()
x = 0
while x < len(self.groups):
tGroup = self.groups[x]
if groupName in tGroup:
del self.groups[x]
didFindGroup = True
x = len(self.groups)
else:
x = x + 1
db.update({'groups': self.groups}, Person.phone == self.phone)
db.close()
def store_user_buy(user, quantity, tx_id):
db = TinyDB(config.DATA_PATH + 'reddit_gold.json')
db.insert({
"user_buyer": user.username,
"quantity": quantity,
"price": config.price_reddit_gold,
"currency": 'pivx',
"amount": config.price_reddit_gold * quantity,
"usd_price": utils.get_coin_value(1, 'pivx', 8),
"total_price": utils.get_coin_value(config.price_reddit_gold * quantity, 'pivx', 2),
'tx_id': tx_id,
'status': "buy",
'time': datetime.datetime.now().isoformat(),
})
db.close()
def set_draft(ids, status=True):
# Status: ['0'] -> publish, ['1'] -> draft_api, ['2'] -> draft
# if status True, set to draft, else set to publish
# given a list of asset_ids, set them to draft or publish depending on status
db = TinyDB('db.json')
Record = Query()
draft = '2' if status else '0'
for item_id in ids:
if item_id:
db.update({'draft_user': draft}, Record.asset_id == item_id)
print(
f"++++++++\nSetting this item: {item_id} to status: {draft}\n++++++++"
)
db.close()
return
def create_doctor(name: str, doc_id: str):
db = TinyDB('scripts/DB/doctor_data_base.json', default_table='doctor')
query = Query()
# check if doctor with this id exist
info_for_given_patient = db.search(query.doc_id == doc_id)
unit = {
"name": name,
"doc_id": doc_id,
}
# insert only if no id match
if len(info_for_given_patient) == 0:
db.insert(unit)
db.close()
def getByName(self, name):
db = TinyDB(self.db_filename)
query = Query()
db_result = db.search(query.name == name)
if len(db_result) == 1:
person = db_result[0]
citizen = self.constructCitizen(person['name'],
person['password'],
person['score'],
person['num_of_ratings'],
person['education'])
db.close()
return citizen
db.close()
return None
def test_caching_json_write(tmpdir):
path = str(tmpdir.join('test.db'))
db = TinyDB(path, storage=CachingMiddleware(JSONStorage))
db.insert({'key': 'value'})
db.close()
# Verify database filesize
statinfo = os.stat(path)
assert statinfo.st_size != 0
# Assert JSON file has been closed
assert db._storage._handle.closed
def test_caching_json_write(tmpdir):
path = str(tmpdir.join('test.db'))
db = TinyDB(path, storage=CachingMiddleware(JSONStorage))
db.insert({'key': 'value'})
db.close()
# Verify database filesize
statinfo = os.stat(path)
assert statinfo.st_size != 0
# Assert JSON file has been closed
assert db._storage._handle.closed
def set_draft(ids, status=True):
# Status: 0 -> publish, 1 -> draft_api, 2 -> draft
# if status True, set to draft, else set to publish
# given a list of asset_ids, set them to draft or publish depending on status
db = TinyDB(cfg.config['db_name'])
Record = Query()
draft = 2 if status else 0
for item_id in ids:
if item_id:
db.update({'draft_user': draft}, Record.asset_id == item_id)
print(
f"++++++++\nSetting this item: {item_id} to status: {draft}\n++++++++"
)
db.close()
return
def test_insert_on_existing_db(tmpdir):
path = str(tmpdir.join('db.json'))
db = TinyDB(path, ensure_ascii=False)
db.insert({'foo': 'bar'})
assert len(db) == 1
db.close()
db = TinyDB(path, ensure_ascii=False)
db.insert({'foo': 'bar'})
db.insert({'foo': 'bar'})
assert len(db) == 3
class Driver:
"""
Holds the driver state so the flasked script can change behaviour based
on what the user injects via HTTP
"""
name = None
base_path = None
storage_path = None
db = None
proxy_host = None
proxy_port = None
def storage(self, storage_path):
self.storage_path = storage_path
def root(self, base_path):
self.base_path = base_path
def start(self, name):
filename = os.path.join(self.base_path,
"{}.yaml".format(name))
if not os.path.exists(filename):
return {"state": "error",
"message": "Ruleset {} not found".format(filename)}
self.name = name
self.db = TinyDB(os.path.join(self.storage_path,
"{}-store.json".format(name)))
return {"ruleset": self.name, "state": "started"}
def stop(self):
if self.name is None:
return {"ruleset": None, "state": None}
message = {"ruleset": self.name, "state": "stopped"}
self.name = None
self.db.close()
self.db = None
return message
def state(self):
if self.name is None:
return {"ruleset": None, "state": None}
return {"ruleset": self.name, "state": "running"}
def test_json_read(tmpdir):
r"""Open a database only for reading"""
path = str(tmpdir.join('test.db'))
with pytest.raises(FileNotFoundError):
db = TinyDB(path, storage=JSONStorage, access_mode='r')
# Create small database
db = TinyDB(path, storage=JSONStorage)
db.insert({'b': 1})
db.insert({'a': 1})
db.close()
# Access in read mode
db = TinyDB(path, storage=JSONStorage, access_mode='r')
assert db.get(where('a') == 1) == {'a': 1} # reading is fine
with pytest.raises(IOError):
db.insert({'c': 1}) # writing is not
db.close()
def start_db(self):
config_yaml = open(self.config_file,'r')
config = yaml.load(config_yaml, Loader=yaml.FullLoader)
config_yaml.close()
print("Hey there, it seems like you are new here. Let's start by creating the database with your admin information.")
admin_name = input('First things first: how can I call you?\n')
admin_mail = input("And what's your email?\n")
admin_rating = input("Which iMDb rating should I store for you (0.0-10.0)?\n")
db = TinyDB(config['DB']['path'])
subscribers = db.table(config['DB']['subscribers'])
admin = {'firstname' : admin_name, 'email': admin_mail, 'admin' : True, 'first' : True, 'rating' : admin_rating}
subscribers.insert(admin)
db.close()
print("Alright, I created the database for you with you as the admin of the service.")
def listvids():
db = TinyDB(DBPATH)
# all = db.all()
table = db.table('vids')
all = table.all()
# c = table.all()
def chunks(l, n):
"""Yield successive n-sized chunks from l."""
for i in range(0, len(l), n):
yield l[i:i + n]
db.close()
return render_template('heidy_check.html',
all=chunks(all, 40),
folder='vids')
def test_empty_cache(self):
"""test un-cached behavior"""
data = forecast_utils.check_prediction_cache(
self.region_id,
self.type_id,
cache_path=self.cache_path
)
assert data is None
assert path.isfile(self.cache_filepath)
tdb = TinyDB(self.cache_filepath)
assert tdb.all() == []
tdb.close()
def insert_people(email, number, name='', token=None):
db = TinyDB("data.json")
People = Query()
if token:
print('Warning: It is not safe to specify the token value. '
'Just let it be None and it will be generated automatically')
else:
token = gen_token(email)
db.insert({
"name": name,
"email": email,
"token": token,
"number": number,
"status": 0
})
db.close()
def log_init():
global RUN_ID, CONFIG
CONFIG = open_config_file()
scriptpath = os.path.realpath(sys.argv[0])
if not os.path.exists(os.path.dirname(DBFILE)):
os.mkdir(os.path.dirname(DBFILE))
#Set up TinyDB database
db = TinyDB(DBFILE, storage=serialization)
guid = str(uuid.uuid4())
# Get env info, etc
run = {"unique_id": guid,
"author": getpass.getuser(),
"description": "",
"inputs": [],
"outputs": [],
"script": scriptpath,
"command": sys.executable,
"environment": [platform.platform(), "python " + sys.version.split('\n')[0]],
"date": datetime.datetime.utcnow()}
if not option(CONFIG, 'ignored metadata', 'git'):
try:
repo = Repo(scriptpath, search_parent_directories=True)
run["gitrepo"] = repo.working_dir
run["gitcommit"] = repo.head.commit.hexsha
run["gitorigin"] = get_origin(repo)
if not option(CONFIG, 'ignored metadata', 'diff'):
whole_diff = ''
diffs = repo.index.diff(None, create_patch=True)
for diff in diffs:
whole_diff += "\n\n\n" + str(diff.diff)
run['diff'] = whole_diff
except:
pass
# Put basics into DB
RUN_ID = db.insert(run)
print("recipy run inserted, with ID %s" % (guid))
db.close()
class NumberStore():
def __init__(self, filename):
self.db = TinyDB(filename)
def initNumber(self, number):
if not self.getNumberDict(number):
self.db.insert({'number': number, 'accesses': [], 'info': '#yolo'})
def touchNumber(self, number):
self.initNumber(number)
#print(self.getNumberDict(number))
#accesses = self.getNumberDict(number)['accesses'].append(datetime.datetime.now())
#self.db.update({'accesses': accesses}, where('number') == number)
def getNumberDict(self, number):
return self.db.get(where('number') == number)
def getNumberList(self):
return (entry['number'] for entry in self.db.all())
def getAccesses(self, number):
# if not number in self.db['numbers']:
# return None
# if not 'info' in self.db['numbers'][number]:
# return None
#
# return self.db['numbers'][number]['info']
return []
def getInfo(self, number):
return self.getNumberDict(number)['info']
def setInfo(self, number, info):
self.initNumber(number)
self.db.update({'info': info}, where('number') == number)
print(self.db.all())
def deleteNumber(self, number):
self.db.remove(where('number') == number)
print(self.db.all())
def close(self):
self.db.close()
def latest_run():
form = SearchForm()
annotateRunForm = AnnotateRunForm()
db = TinyDB(recipyGui.config.get('tinydb'))
runs = db.all()
runs = sorted(runs, key = lambda x: parse(x['date'].replace('{TinyDate}:', '')), reverse=True)
r = db.get(eid=runs[0].eid)
diffs = db.table('filediffs').search(Query().run_id == r.eid)
db.close()
return render_template('details.html', query='', form=form, run=r,
annotateRunForm=annotateRunForm,
dbfile=recipyGui.config.get('tinydb'), diffs=diffs,
active_page='latest_run')
class FolderManager:
def __init__(self, path):
self.db_file = os.path.join(path, CONF_DIR, FOLDER_DB_FN)
self._db = TinyDB(self.db_file)
def open_db(self):
self._db = TinyDB(self.db_file)
def close_db(self):
self._db.close()
def add_folder(self, file_name):
if not self.folder_exists(file_name):
entry = {'file_name': file_name}
self._db.insert(entry)
def get_all_entries(self):
return self._db.all()
def folder_exists(self, file_name):
""" checks if a folder has been added """
entries = self._db.search(where('file_name') == file_name)
if entries:
return True
else:
return False
def remove_element(self, file_name):
self._db.remove(where('file_name') == file_name)
def get_file_names(self):
""" returns all the file names of folders that the user has added """
file_names = []
for entry in self._db.all():
file_names.append(entry['file_name'])
return file_names
def get_folder_by_name(self, expected_name):
""" get documents by the specified property """
entry = self._db.get(where('file_name') == expected_name)
return entry
def clear_all(self):
self._db.purge()
def get_corpus_path(name: str) -> [str, None]:
"""
Get corpus path
:param string name: corpus name
"""
db = TinyDB(corpus_db_path())
temp = Query()
if len(db.search(temp.name == name)) > 0:
path = get_full_data_path(db.search(temp.name == name)[0]["file"])
db.close()
if not os.path.exists(path):
download(name)
return path
return None
def search(args):
if not os.path.exists(os.path.dirname(DBFILE)):
os.mkdir(os.path.dirname(DBFILE))
db = TinyDB(DBFILE)
filename = args['<outputfile>']
if args['--fuzzy']:
results = db.search(where('outputs').any(lambda x: re.match(".+%s.+" % filename, x)))
elif args['--regex']:
results = db.search(where('outputs').any(lambda x: re.match(filename, x)))
else:
results = db.search(where('outputs').any(os.path.abspath(filename)))
def change_date(result):
result['date'] = result['date'].replace('{TinyDate}:', '')
return result
results = [change_date(result) for result in results]
# Sort the results
results = sorted(results, key = lambda x: parse(x['date']))
if len(results) == 0:
print("No results found")
else:
if args['--all']:
for r in results:
print_result(r)
print("-"*40)
else:
print_result(results[-1])
if len(results) > 1:
print("** Previous runs creating this output have been found. Run with --all to show. **")
if args['--diff']:
if 'diff' in results[-1]:
print("\n\n")
print(results[-1]['diff'])
db.close()
def scrap(url):
db = TinyDB(DB_FILENAME)
response = requests.get(url)
soup = BeautifulSoup(response.text, 'html.parser')
# Remove promoted section
soup.select_one('div.view.top-listings').extract()
# Remove reply window
soup.select_one('li.result.reply').extract()
# Find all results
results = []
for li in soup.select('li.result'):
results.append(_parse_result(li))
# Skip those that are already there - single result found will break out
# of the loop
valid = []
for result in results:
if db.contains(_prepare_query(result)):
break
db.insert(_to_dict(result))
valid.append(result)
db.close()
return valid
class DatabaseAccess(object):
'''
Raw database access. This module might be unnecessary, it's just a simple
wrapper around TinyDB. It doesn't isolate the Query abstractions of TinyDB
so we don't get any portability or independence from TinyDB. But it is a
place to change storage, middleware, etc.
'''
def __init__(self, filename):
'''
Initialize a database with a file.
:param filename: database filename.
'''
self.filename = str(filename)
self.db = TinyDB(self.filename, storage=CachingMiddleware(JSONStorage))
def close(self):
'''Close the database file.'''
self.db.close()
def table(self, tableName):
return self.db.table(tableName)
# def insert(self, *args, **kwargs):
# '''Insert an object into the database.'''
# return self.db.insert(*args, **kwargs)
# def get(self, *args, **kwargs):
# '''Get an object from the database.'''
# return self.db.Get(*args, **kwargs)
# def contains(self, *args, **kwargs):
# '''Test an object from the database.'''
# return self.db.Get(*args, **kwargs)
# def search(self, *args, **kwargs):
# '''Query for objects.'''
# return self.db.search(*args, **kwargs)
# def remove(self, *args, **kwargs):
# '''Remove objects by query.'''
# return self.db.remove(*args, **kwargs)
def setSingleton(self, kind, model):
id = None
self.db.table(kind).purge()
id = self.db.table(kind).insert(model)
return id
def getSingleton(self, kind):
objs = self.db.table(kind).all()
if len(objs) == 0:
return None
return objs[0]
def getArgs(self):
return self.getSingleton('fashion.prime.args')
def isVerbose(self):
args = self.getArgs()
if args is None:
return False
return args["verbose"]
def isDebug(self):
args = self.getArgs()
if args is None:
return False
return args["debug"]
def kinds(self):
k = self.db.tables()
k.remove("_default")
return k
def log_update(field, filename, source):
filename = os.path.abspath(filename)
print("Adding %s to %s using $s" % (field, filename, source))
db = TinyDB(DBFILE)
db.update(append(field, filename), eids=[RUN_ID])
db.close()
