Example #1
1
class DB:
    def __init__(self,db_path):
        self.db = TinyDB(db_path)

    def add(self, data):
        # Only add it if you can't find it
        Track = Query()
        if not self.db.get(Track.display_id == data['display_id']):
            return self.db.insert(data)

    def searchById(self, video_id):
        Track = Query()
        return self.db.get(Track.display_id == video_id)

    def search(self, text):
        pattern = re.compile(text,re.IGNORECASE)
        def test(txt):
            return pattern.search(txt)

        Track = Query()
        q = Track.title.test(test) | Track.description.test(test)
        return self.db.search(q)

    def all(self):
        return self.db.all()
Example #2
0
def index():
    form = SearchForm()

    query = request.args.get('query', '').strip()

    db = TinyDB(recipyGui.config.get('tinydb'))

    if not query:
        runs = db.all()
    else:
        # Search run outputs using the query string
        runs = db.search(
            where('outputs').any(lambda x: listsearch(query, x)) |
            where('inputs').any(lambda x: listsearch(query, x)) |
            where('script').search(query) |
            where('notes').search(query) |
            where('unique_id').search(query))
    runs = sorted(runs, key = lambda x: parse(x['date'].replace('{TinyDate}:', '')) if x['date'] is not None else x['eid'], reverse=True)

    run_ids = []
    for run in runs:
        if 'notes' in run.keys():
            run['notes'] = str(escape(run['notes']))
        run_ids.append(run.eid)

    db.close()

    return render_template('list.html', runs=runs, query=query, form=form,
                           run_ids=str(run_ids),
                           dbfile=recipyGui.config.get('tinydb'))
Example #3
0
def test_serialisation_of_pandas_dataframe(tmpdir):
    from sacred.observers.tinydb_hashfs import (DataFrameSerializer,
                                                SeriesSerializer)
    from tinydb_serialization import SerializationMiddleware

    import numpy as np
    import pandas as pd

    # Setup Serialisation object for non list/dict objects
    serialization_store = SerializationMiddleware()
    serialization_store.register_serializer(DataFrameSerializer(),
                                            'TinyDataFrame')
    serialization_store.register_serializer(SeriesSerializer(),
                                            'TinySeries')

    db = TinyDB(os.path.join(tmpdir.strpath, 'metadata.json'),
                storage=serialization_store)

    df = pd.DataFrame(np.eye(3), columns=list('ABC'))
    series = pd.Series(np.ones(5))

    document = {
        'foo': 'bar',
        'some_dataframe': df,
        'nested': {
            'ones': series
        }
    }

    db.insert(document)
    returned_doc = db.all()[0]

    assert returned_doc['foo'] == 'bar'
    assert (returned_doc['some_dataframe'] == df).all().all()
    assert (returned_doc['nested']['ones'] == series).all()
Example #4
0
def dragon_greet():
    print("_______________________________________________________________\n")
    time = datetime.datetime.now().time()

    global user_full_name
    global user_prefix
    global config_file

    command = "getent passwd $LOGNAME | cut -d: -f5 | cut -d, -f1"
    user_full_name = os.popen(command).read()
    user_full_name = user_full_name[:-1]  # .decode("utf8")
    home = expanduser("~")
    config_file = TinyDB(home + '/.dragonfire_config.json')
    callme_config = config_file.search(Query().datatype == 'callme')
    if callme_config:
        user_prefix = callme_config[0]['title']
    else:
        gender_config = config_file.search(Query().datatype == 'gender')
        if gender_config:
            user_prefix = GENDER_PREFIX[gender_config[0]['gender']]
        else:
            gender = Classifier.gender(user_full_name.split(' ', 1)[0])
            config_file.insert({'datatype': 'gender', 'gender': gender})
            user_prefix = GENDER_PREFIX[gender]

    if time < datetime.time(12):
        time_of_day = "morning"
    elif datetime.time(12) < time < datetime.time(18):
        time_of_day = "afternoon"
    else:
        time_of_day = "evening"
    userin.execute(["echo"], "To activate say 'Dragonfire!' or 'Wake Up!'")
    userin.say(" ".join(["Good", time_of_day, user_prefix]))
Example #5
0
    def AddUser(self, username, chatid):
        my_pokemon = [0] * 152 # Matching arr index to pokemon index (0 is disregarded)

        db = TinyDB('users.json')
        db.insert({'username': username, 'chatid': chatid, 'pokemon': my_pokemon})
        
        pass # RETURN: check bool
Example #6
0
def check_prediction_cache(
        region_id,
        type_id,
        cache_path=CACHE_PATH,
        db_filename='prophet.json'
):
    """check tinyDB for cached predictions

    Args:
        region_id (int): EVE Online region ID
        type_id (int): EVE Online type ID
        cache_path (str): path to caches
        db_filename (str): name of tinydb

    Returns:
        pandas.DataFrame: cached prediction

    """
    utc_today = datetime.utcnow().strftime('%Y-%m-%d')

    prediction_db = TinyDB(path.join(cache_path, db_filename))

    raw_data = prediction_db.search(
        (Query().cache_date == utc_today) &
        (Query().region_id == region_id) &
        (Query().type_id == type_id)
    )

    prediction_db.close()

    if raw_data:
        panda_data = pd.read_json(raw_data[0]['prediction'])
        return panda_data
    else:
        return None
Example #7
0
def insert_test(db_file='db.json'):
    db = TinyDB(db_file)
    db.insert({
        'name': 'Aman Verma',
        'items': 1,
        'contact': 7890701597
    })
Example #8
0
def test_json_readwrite(tmpdir):
    """
    Regression test for issue #1
    """
    path = str(tmpdir.join('test.db'))

    # Create TinyDB instance
    db = TinyDB(path, storage=JSONStorage)

    item = {'name': 'A very long entry'}
    item2 = {'name': 'A short one'}

    get = lambda s: db.get(where('name') == s)

    db.insert(item)
    assert get('A very long entry') == item

    db.remove(where('name') == 'A very long entry')
    assert get('A very long entry') is None

    db.insert(item2)
    assert get('A short one') == item2

    db.remove(where('name') == 'A short one')
    assert get('A short one') is None
Example #9
0
def update_statistic():
    db = TinyDB("data/db.json")
    rows = db.all()
    rows.sort(key=lambda x: int(x['rid']), reverse=True)
    levels = {1: [], 2: [], 3: []}
    for row in rows:
        levels[row['level']].append(row)

    out = []
    header = '|%-35s|%-40s|%-6s|%-10s|%5s|%10s|' % ("round", "problem", "solved", "ave_pts",
            "rate", "submission")
    for k, v in levels.items():
        out.append('# LEVEL %d' % k)
        out.append('-' * len(header))
        out.append(header)
        out.append('-' * len(header))
        v.sort(key=lambda x: difficulty(x), reverse=True)
        for i in v:
            out.append('|%-35s|%-40s|%-6s|%-10.2lf|%-4.3lf|%10s|' % (i['round'], i['name'], i['solved'],
                i['average_pts'], i['correct_rate'], i['submissions']))
        out.append('*' * len(header))
    with open("data/statistic.txt", 'w') as f:
        for i in out:
            f.write(i + '\n')
    print '>>>> data/statistic.txt has been updated.'
Example #10
0
    def __init__(self, path):

        root_dir = os.path.abspath(path)
        if not os.path.exists(root_dir):
            raise IOError('Path does not exist: %s' % path)

        fs = HashFS(os.path.join(root_dir, 'hashfs'), depth=3,
                    width=2, algorithm='md5')

        # Setup Serialisation for non list/dict objects
        serialization_store = SerializationMiddleware()
        serialization_store.register_serializer(DateTimeSerializer(),
                                                'TinyDate')
        serialization_store.register_serializer(FileSerializer(fs),
                                                'TinyFile')
        if opt.has_numpy:
            serialization_store.register_serializer(NdArraySerializer(),
                                                    'TinyArray')
        if opt.has_pandas:
            serialization_store.register_serializer(DataFrameSerializer(),
                                                    'TinyDataFrame')
            serialization_store.register_serializer(SeriesSerializer(),
                                                    'TinySeries')

        db = TinyDB(os.path.join(root_dir, 'metadata.json'),
                    storage=serialization_store)

        self.db = db
        self.runs = db.table('runs')
        self.fs = fs
    def test_write_first_cache(self):
        """test write behavior on first pass (cache-buster mode)"""
        self.test_clear_existing_cache()    #blowup existing cache again

        dummy_data = forecast_utils.parse_emd_data(DEMO_DATA['result'])

        forecast_utils.write_prediction_cache(
            self.region_id,
            self.type_id,
            dummy_data,
            cache_path=self.cache_path
        )

        assert path.isfile(self.cache_filepath)

        tdb = TinyDB(self.cache_filepath)

        data = tdb.all()[0]

        keys_list = [
            'cache_date',
            'region_id',
            'type_id',
            'lastWrite',
            'prediction'
        ]
        assert set(keys_list) == set(data.keys())
        dummy_str_data = dummy_data.to_json(
            date_format='iso',
            orient='records'
        )
        cached_data = pd.read_json(data['prediction'])

        assert data['prediction'] == dummy_str_data
        tdb.close()
Example #12
0
def test_serialisation_of_numpy_ndarray(tmpdir):
    from sacred.observers.tinydb_hashfs import NdArraySerializer
    from tinydb_serialization import SerializationMiddleware
    import numpy as np

    # Setup Serialisation object for non list/dict objects
    serialization_store = SerializationMiddleware()
    serialization_store.register_serializer(NdArraySerializer(), 'TinyArray')

    db = TinyDB(os.path.join(tmpdir.strpath, 'metadata.json'),
                storage=serialization_store)

    eye_mat = np.eye(3)
    ones_array = np.ones(5)

    document = {
        'foo': 'bar',
        'some_array': eye_mat,
        'nested': {
            'ones': ones_array
        }
    }

    db.insert(document)
    returned_doc = db.all()[0]

    assert returned_doc['foo'] == 'bar'
    assert (returned_doc['some_array'] == eye_mat).all()
    assert (returned_doc['nested']['ones'] == ones_array).all()
Example #13
0
class pcDB:
    def __init__(self,table="default"):
        #'/path/to/db.json'
        path=''
        self.table=table
        self.db = TinyDB(path).table(table)
    def insert(self,_dict):
        '''
        :return:
        '''
        self.db.insert(_dict)
        # db.insert({'int': 1, 'char': 'a'})
        # db.insert({'int': 1, 'char': 'b'})
        pass
    def getAll(self):
        '''
        not param just get all data
        :return:
        '''
        return self.db.all()
        #db.search()
        pass
    pass
#
# from tinydb.storages import JSONStorage
# from tinydb.middlewares import CachingMiddleware
# db = TinyDB('/path/to/db.json', storage=CachingMiddleware(JSONStorage))
Example #14
0
def get_coffees(active):
  name = ''
  name_link = ''
  nav_links = []
  description = ''
  db = TinyDB('db/coffee.json')
  coffee = db.all()
  count = 0
  coffee_id = 0
  for i in coffee:
    if active == -1 and count == 0:
      nav_links.append(("/"+str(i['id']),"active",i['name']))
      name = i['name']
      description = i['description']
      coffee_id = i['id']
      name_link = '/'+str(i['id'])
    elif active == -1 and count > 0:
      nav_links.append(("/"+str(i['id']),"",i['name']))
    elif active == i['id']:
      nav_links.append(("/"+str(i['id']),"active",i['name']))
      name = i['name']
      description = i['description']
      coffee_id = i['id']
      name_link = '/'+str(i['id'])
    else:
      nav_links.append(("/"+str(i['id']),"",i['name']))
    count = count+1
  for i in nav_links:
    print i
  print name
  print name_link

  return nav_links, name, name_link, description, coffee_id
Example #15
0
def test_process(tmpdir, capsys, side_effect, success):
    db_fn = tmpdir.join('database.json')
    log_dir = tmpdir.mkdir('log')
    db = TinyDB(db_fn.strpath)
    process_me = '/my/path/A12345'
    accession_number = op.basename(process_me)
    paths2process = {process_me: 42}
    with patch('subprocess.Popen') as mocked_popen:
        stdout = b"INFO: PROCESSING STARTS: {'just': 'a test'}"
        mocked_popen_instance = mocked_popen.return_value
        mocked_popen_instance.side_effect = side_effect
        mocked_popen_instance.communicate.return_value = (stdout, )
        # set return value for wait
        mocked_popen_instance.wait.return_value = 1 - success
        # mock also communicate to get the supposed stdout
        process(paths2process, db, wait=-30, logdir=log_dir.strpath)
        out, err = capsys.readouterr()
        log_fn = log_dir.join(accession_number + '.log')

        mocked_popen.assert_called_once()
        assert log_fn.check()
        assert log_fn.read() == stdout.decode('utf-8')
        assert db_fn.check()
        # dictionary should be empty
        assert not paths2process
        assert out == 'Time to process {0}\n'.format(process_me)

        # check what we have in the database
        Path = Query()
        query = db.get(Path.input_path == process_me)
        assert len(db) == 1
        assert query
        assert query['success'] == success
        assert query['accession_number'] == op.basename(process_me)
        assert query['just'] == 'a test'
Example #16
0
 def setUp(self):
     with open(os.path.join(BASE_DIR, 'tests', 'fixtures', 'test.json')) as f:
         self.j = json.load(f)
     self.database_name = os.path.join(os.getcwd(), 'test.db')
     db = TinyDB(self.database_name)
     db.insert_multiple(self.j)
     db.close
Example #17
0
def crawl(sr=0, er=3):
    archive = dict()
    url = "https://community.topcoder.com/tc?module=ProblemArchive&sr=%d&er=%d" % (sr, er)
    print "requesting seed page..."
    r = requests.get(url)
    html = h.unescape(r.content.decode('utf-8'))
    doc = pq(html)
    for i in doc('table.paddingTable2').eq(2).children()[3:]:
        round_name = pq(i).children().eq(2).find('a').text()
        sub_url = pq(i).children().eq(2).find('a').attr.href
        if sub_url is not None:
            rid = sub_url.split('rd=')[-1]
            archive[round_name] = {'rid': rid, 'round': round_name}
    db = TinyDB("data/db.json")
    tot = len(archive.values())
    cur = 0
    prob_cnt = 0
    for k in archive.values():
        problems = crawl_round(k['rid'], k['round'])
        print 'parse result:'
        for p in problems:
            for pk, pv in p.items():
                print "%-15s:   %s" % (pk, pv)
            prob_cnt += 1
            q = Query()
            if not db.search(q.name == p['name']):
                print '>>>>>>> insert problem: %s' % p['name']
                db.insert(p)
            print '-' * 10
        cur += 1
        print '*' * 10, 'finish', k['round'], ',tot rounds:', tot, 'cur round:', cur, 'round problems:', len(problems), '*' * 10
    print 'done, total round: %d, total problems: %d' % (cur, prob_cnt)
Example #18
0
    def __init__(self):
        self.sentCache = {}
        self.checkingThread = threading.Thread(target = self.startThread)
        #self.checkingThread.daemon = True
        self.keepChecking = True
        self.config = Config()

        if self.config.winning_streak:
            self.winning_streak_messages = self.read_spree_file(self.config.winning_streak_file)
        if self.config.losing_streak:
            self.losing_streak_messages = self.read_spree_file(self.config.losing_streak_file)

        # Debug purposes
        self.printOutput = False

        # Initializing the API
        key = dotamatch.get_key()  # Steam Dev Key (~/.steamapi)
        try:
            self.match_history = MatchHistory(key)
            self.match_details = MatchDetails(key)
            self.account_details = PlayerSummaries(key)
            self.heroes = Heroes(key).heroes()

            # ActualDB
            db = TinyDB(self.config.db_path)
            #db.purge()
            #db.purge_tables()

            self.matches_table = db.table('matches')
            self.matches_info_table = db.table('matches_info')
        except dotamatch.api.ApiError:
            print u"Erro ao conectar à API."
Example #19
0
def prelimsearchURLs():
    db = TinyDB('db.json')
    prelimtable = db.table('prelimcasedetails')
    cases = getAllSearchResults()
    prelimtable.purge()
    for idx, case in enumerate(cases):
        prelimtable.insert({'caseid': case.caseid, 'casename': case.casename, 'prelimvideourl': case.videourl,'detailedVideoURL':'0'})
Example #20
0
def is_vip(author):
	vip_db = TinyDB("dbs/vips.json")
	try:
		vip_db.get(where("email") == author)
		return True
	except:
		return False
Example #21
0
  def save_customer(self, customer):

    if not self.validate(customer):
      return None
    
    db = TinyDB('db/db.json')
    return db.insert(customer)
Example #22
0
def process_and_add_one(pdf_path):
    pdf_name = pdf_path.split("/")
    pdf_name = pdf_name[-1]
    directory = pdf_path[0 : -len(pdf_name)]
    stripped_name = pdf_name[0:-4]
    title_path = title_dir + "/" + stripped_name + ".xml"
    extract_title(pdf_path, title_path)
    # check if title extraction worked, otherwise stop with this one
    tf = open(title_path, "r")
    txml = tf.read()
    if txml == "title extraction failed":
        return None

    # build dictionary with info we've got
    tf = open(title_path, "r")
    txml = tf.read()
    txml = txml.split(">")
    title = "title not found"
    for line in txml:
        if "</title" in line:
            title = line[0:-7]
            print title
            break

    # save nice text version of title
    txt_name_path = title_path[0:-4] + ".txt"
    ftxt = open(txt_name_path, "a")
    ftxt.write(title)
    if title == "title not found":
        return None

    # if title was found, get DOI from it
    currDOI = get_DOI_from_title(title)
    # open/create tiny db
    db = TinyDB(db_loc)
    # make sure the paper isnt in the db already
    paper = Query()
    gotit = db.search(paper.ownDOI == currDOI)
    if gotit:
        return currDOI

    text_path_xml = text_dir + "/" + stripped_name + ".xml"
    text_path_txt = text_dir + "/" + stripped_name + ".txt"
    if not extract_text(pdf_path, text_path_xml, text_path_txt):
        print ("text extraction failed")
        return None

    # only extract bibtex if you don't have it already, because this is the long part
    # TODO: Return before doing bib extraction
    bib_path = bib_dir + "/" + stripped_name + ".bib"
    if not extract_bibtex(pdf_path, bib_path):
        print ("caught in the new code")
        return None

    refDOIs = get_ref_list_DOIs(bib_path)

    new_dict = {"ownDOI": currDOI, "refDOIs": refDOIs, "filename": stripped_name}
    db.insert(new_dict)
    return currDOI
Example #23
0
def write(message, args):
    # message.reply('I can understand hi or HI!')
    # react with thumb up emoji
    #message.react('+1')
    db = TinyDB('db.json')
    db.insert({'value': args});
    print args
    db.close()
Example #24
0
def log_output(filename, source):
    filename = os.path.abspath(filename)
    if option(CONFIG, 'general', 'debug'):
        print("Output to %s using %s" % (filename, source))
    #Update object in DB
    db = TinyDB(DBFILE)
    db.update(append("outputs", filename), eids=[RUN_ID])
    db.close()
Example #25
0
class EventModel(object):
    def __init__(self, uri):
        self.uri = uri
        self.db = None
        self.reload()

    def reload(self):
        if self.db is not None:
            self.db.close()
        self.db = TinyDB(self.uri, indent=2)
        self.actions = self.db.table('actions')
        self.alarms = self.db.table('alarms')

    def get_action_by_id(self, action_id):
        return self.actions.get(eid=action_id)

    def get_alarm_by_id(self, alarm_id):
        return self.alarms.get(eid=alarm_id)

    def get_actions_by_alarm(self, alarm):
        for action_id in alarm.get('actions', []):
            action = self.get_action_by_id(action_id)
            if action is None: continue
            yield action

    def get_all_alarms(self):
        return self.alarms.all()

    def get_all_actions(self):
        return self.actions.all()

    def get_all_alarms_expanded(self):
        for alarm in self.get_all_alarms():
            for action in self.get_actions_by_alarm(alarm):
                yield alarm, action

    def add_event(self, alarm, actions):
        action_ids = [self.add_action(a) for a in actions]
        alarm['actions'] = action_ids
        return self.alarms.insert(alarm)

    def add_action(self, action):
        return self.actions.insert(action)

    def add_alarm(self, alarm):
        return self.add_event(alarm, [])

    def update_alarm(self, alarmid, new_fields={}):
        return self.alarms.update(new_fields, eids=[alarmid])

    def update_action(self, actionid, new_fields={}):
        return self.actions.update(new_fields, eids=[actionid])

    def delete_alarm(self, alarmid):
        return self.alarms.remove(eids=[alarmid])

    def delete_action(self, actionid):
        return self.actions.remove(eids=[actionid])
Example #26
0
def get_file(name):
    db=TinyDB(path_db_)
    temp = Query()
    if len(db.search(temp.name==name))>0:
        path= get_path_data(db.search(temp.name==name)[0]['file'])
        db.close()
        if not os.path.exists(path):
            download(name)
        return path
Example #27
0
def get_genres(database_name):
    """Utility method to get all the genres as a set"""
    db = TinyDB(os.path.join(os.getcwd(), database_name))
    all_genres = { song['genre'] for song in db.all() }
    specific_genres = set()
    for genre in all_genres:
        specific_genres = specific_genres.union(set(genre.strip().split('/')))
    db.close()
    return _strip_spaces(specific_genres)
Example #28
0
def test_upgrade(tmpdir):
    db_file = tmpdir.join('db.json')
    db_file.write(v1_0)

    # Run upgrade
    assert migrate(str(db_file)) is True
    db = TinyDB(str(db_file))

    assert db.count(where('key') == 'value') == 1
Example #29
0
def test_json_kwargs(tmpdir):
    db_file = tmpdir.join('test.db')
    db = TinyDB(str(db_file), sort_keys=True, indent=4, separators=(',', ': '))

    # Write contents
    db.insert({'b': 1})
    db.insert({'a': 1})

    assert db_file.read() == '''{
Example #30
0
def api():
    db = TinyDB(DB_FILENAME)
    results = db.search((Query().hidden == False) | (Query().starred == True))
    db.close()
    for result in results:
        result['eid'] = result.eid
        result['url'] = url_for('goto', eid=result.eid)
    results.sort(key=lambda r: r['created_at'])
    return json.dumps(results)
Example #31
0
    file_list = glob.glob("/home/feng/桌面/to_be_calculated/" + str(patient[i]) +
                          "/score/" + measure + "/*.txt")
    op_file = "Clinical_Note/" + str(patient[i]) + "/" + str(
        patient[i]) + "_scoredb_" + measure + ".txt"
    # 將某病人某方法算出的全部smt相似度分數都聚合在一起
    print("正在結合所有相似度分數 for", patient[i], ".....")
    if True:
        with open(op_file, 'w') as outfile:
            for fname in file_list:
                with open(fname) as infile:
                    outfile.write(infile.read())

    from tinydb import TinyDB
    from tinydb.queries import Query, where
    print("正在建立分數資料庫 for", patient[i], ".....")
    db = TinyDB("Clinical_Note/" + str(patient[i]) + "/DB/" + str(patient[i]) +
                "_scoredb_path.json")
    f = open("Clinical_Note/" + str(patient[i]) + "/" + str(patient[i]) +
             "_scoredb_path.txt")
    rec = f.readline()
    # # print(rec.split("<>")[0], rec.split("<>")[1], rec.split("<>")[2].replace('\n', ''))
    # q = Query()
    # res = db.search((q.cui1 == 'C0748646') & (q.cui2 == 'C0010520'))

    # # print(res[0]['score'])
    # print(res)

    # 建立DB階段
    if True:
        cnt = 0
        while rec:
            db.insert({