コード例 #1
0
def run_migration():
    migrator = SqliteMigrator(db)
    with db.atomic():
        migrate(migrator.drop_column('note', 'reminder_task_created'), )
        FTSNote.create_table(True)

    with db.atomic():
        for note in Note.select():
            FTSNote.store_note(note)
コード例 #2
0
ファイル: cmp.py プロジェクト: IncreikeSoul/scraping-web
def process_from_csv(input_file):
    stream = io.StringIO(input_file.read().decode('utf-8'))
    csvfile = csv.reader(stream)
    inserted = 0
    invalid = 0
    duplicated = 0
    records = []

    for row in csvfile:
        try:
            cmp = row[0].strip().upper()
            if len(cmp) == 6:
                with db.atomic():
                    doctor = Doctor.create(id=cmp)
                # Add to process queue
                result = default_queue.enqueue(scrap_cmp, doctor)
                inserted += 1
            else:
                invalid += 1
        except peewee.IntegrityError:
            duplicated += 1
        except Exception as e:
            invalid += 1

    return inserted, duplicated, invalid
コード例 #3
0
ファイル: sunedu.py プロジェクト: IncreikeSoul/scraping-web
def process_from_csv(input_file):
    stream = io.StringIO(input_file.read().decode('utf-8'))
    csvfile = csv.reader(stream)
    inserted = 0
    invalid = 0
    duplicated = 0
    records = []

    for row in csvfile:
        try:
            document_number = row[0].strip().upper()
            if len(document_number) == 8:
                with db.atomic():
                    graduate = Graduate.create(id=document_number)
                # Add to process queue
                result = default_queue.enqueue(scrap_document_number, graduate)
                inserted += 1
            else:
                invalid += 1
        except peewee.IntegrityError:
            duplicated += 1
        except Exception as e:
            invalid += 1

    return inserted, duplicated, invalid
コード例 #4
0
    def save(self, *args, **kwargs):
        # Split out the text content and any tasks.
        self.content, tasks = self.parse_content_tasks()

        # Update the timestamp.
        self.timestamp = datetime.datetime.now()

        # Save the note.
        with db.atomic():
            ret = super(Note, self).save(*args, **kwargs)

            # Store the tasks in the database.
            if tasks:
                Task.delete().where(Task.note == self).execute()
                for idx, (finished, title) in enumerate(tasks):
                    Task.create(
                        note=self,
                        finished=finished,
                        title=title,
                        order=idx)

            # Store the content for full-text search.
            FTSNote.store_note(self)

        return ret
コード例 #5
0
def populate_entities(entities_list):
    with db.atomic():
        for i, entity in enumerate(entities_list):
            try:
                entity_name = "{}.{}".format(
                    entity["inchikey"].replace("InChIKey=", ""), "json")
                if i % 100 == 0:
                    print(entity_name, "populating", i)
                ClassyFireEntity.get_or_create(inchikey=entity_name,
                                               responsetext=json.dumps(entity),
                                               status="DONE")
            except KeyboardInterrupt:
                raise
            except:
                print("ERROR", entity)
コード例 #6
0
ファイル: pay_run.py プロジェクト: Davincier/payrun
    def make_diffs(self):
        from models import PayRecord, PayDiff
        from app import db

        prev_rex = self._get_previous_records()
        if not prev_rex:
            return
        for current_rec in self.records:
            qry = prev_rex.select().where(PayRecord.employee == current_rec.employee)
            if not qry.exists():
                continue
            rec_diffs = PayRecord.get_diffs(prev_rex.get(), current_rec)
            if rec_diffs:
                with db.atomic():
                    PayDiff.insert_many(rec_diffs).execute()
コード例 #7
0
ファイル: import_data.py プロジェクト: Davincier/payrun
def import_data():
    path = 'c:\\bench\\payrun\\pyvista\\fms\\tests\\run'
    runs = [
        {'ien': '559', 'fy': 13, 'pp': 15, 'cp': '015'},
        {'ien': '559', 'fy': 13, 'pp': 15, 'cp': '016'},
        {'ien': '560', 'fy': 13, 'pp': 16, 'cp': '015'},
        {'ien': '560', 'fy': 13, 'pp': 16, 'cp': '016'},
        {'ien': '616', 'fy': 15, 'pp': 20, 'cp': '015'},
        {'ien': '616', 'fy': 15, 'pp': 20, 'cp': '016'},
        {'ien': '617', 'fy': 15, 'pp': 21, 'cp': '015'},
        {'ien': '617', 'fy': 15, 'pp': 21, 'cp': '016'},
        {'ien': '618', 'fy': 15, 'pp': 22, 'cp': '015'},
        {'ien': '618', 'fy': 15, 'pp': 22, 'cp': '016'},
        {'ien': '619', 'fy': 15, 'pp': 23, 'cp': '015'},
        {'ien': '619', 'fy': 15, 'pp': 23, 'cp': '016'}
    ]
    with db.atomic():
        PayRun.insert_many(runs).execute()

    runs = [run for run in PayRun.select()]

    for run in runs:
        print('%d: %s: %s' % (run.id, run.ien, str(run)))
        filename = path + run.ien + '_' + run.cp + '.txt'
        f = open(filename)
        dta = f.readlines()
        f.close()

        for line in dta:
            if line[0].isdigit():
                emp = make_emp(line, run.cp)
                rec = make_rec(line.rstrip(), run.id, emp.id)

        prev_run_id = run.get_previous_run_id()
        if prev_run_id:
            print('getting rex...')
            rex = [rec for rec in run.records]
            print('%d rex' % len(rex))
            for rec in rex:
                print('rec: %d' % rec.id)
                rec.make_diffs(prev_run_id)
コード例 #8
0
ファイル: tasks.py プロジェクト: Catgroove/dotaninja
def add_match(match_id):
    """Queries a match from the Steam API and adds it to the database.
    Returns True if it succeeds or if match already exists, or False if it fails.
    """
    match = api.get_match_details(match_id)["result"]

    if "error" in match:
        return False

    try:
        new_match = Match.create(**match)
    except IntegrityError:
        return True
    except PyDotaError:
        return False

    player_list = [dict(match_id=new_match.match_id, **player) for player in match["players"]]

    with db.atomic():
        MatchPlayer.insert_many(player_list).execute()

    return True
コード例 #9
0
ファイル: pay_record.py プロジェクト: Davincier/payrun
    def make_diffs(self, prev_run_id):
        from app import db, diff_fields
        from models import PayDiff

        prev_rec = self._get_previous_record(prev_run_id)
        if not prev_rec:
            return

        diffs = []
        for field in diff_fields[3:]:
            curval = getattr(self, field)
            preval = getattr(prev_rec, field)
            if curval != preval:
                diffs.append({
                    'payrun': self.payrun,
                    'record': self,
                    'field_name': field,
                    'current_amount': curval,
                    'previous_amount': preval
                })
        if diffs:
            with db.atomic():
                PayDiff.insert_many(diffs).execute()
コード例 #10
0
def process_from_csv(input_file):
    stream = io.StringIO(input_file.read().decode('utf-8'))
    csvfile = csv.reader(stream, delimiter=';')
    inserted = 0
    invalid = 0
    duplicated = 0
    records = []

    for row in csvfile:
        try:
            document_ruc = row[0].strip().upper()
            document_dni = row[1].strip().upper()

            if len(document_ruc) == 11 and (len(document_dni) > 7
                                            and len(document_dni) < 10):
                with db.atomic():
                    rrll_claro = RRLL.create(ruc=document_ruc,
                                             dni=document_dni,
                                             provider='CLARO')
                    rrll_movistar = RRLL.create(ruc=document_ruc,
                                                dni=document_dni,
                                                provider='MOVISTAR')
                # Add to process queue
                result = default_queue.enqueue(scrap_claro_lines, rrll_claro)
                result = default_queue.enqueue(scrap_movistar_lines,
                                               rrll_movistar)
                inserted += 1
            else:
                invalid += 1

        except peewee.IntegrityError:
            duplicated += 1
        except Exception as e:
            print(e)
            invalid += 1

    return inserted, duplicated, invalid
コード例 #11
0
ファイル: quoteimport.py プロジェクト: agateau/tmc2
def do_import(quotes):
    lst = create_insert_list(quotes)
    with db.atomic():
        Quote.insert_many(lst).execute()
コード例 #12
0
ファイル: import.py プロジェクト: Debetux/CodesPostauxAPI
import csv
import os
import time
from models import City
from app import db


City.create_table(True)
i = 0
data_source = []
print(City.delete().execute())
with open("cp.csv", "r", encoding="utf-8") as csvfile:
    spamreader = csv.reader(csvfile, delimiter=";")

    for row in spamreader:
        data_source.append({"insee": row[0], "name": row[1], "postal_code": row[2], "label": row[3]})
        i += 1

        if len(data_source) > 100:
            print("+1")
            with db.atomic():
                City.insert_many(data_source).execute()
            data_source = []
            time.sleep(0.02)
コード例 #13
0
import os
import time
from models import City
from app import db


City.create_table(True)
i = 0
data_source = []
print(City.delete().execute())
with open('cp.csv', 'r', encoding="utf-8") as csvfile:
    spamreader = csv.reader(csvfile, delimiter=';')

    for row in spamreader:
        data_source.append(
            {
                "insee": row[0],
                "name": row[1],
                "postal_code": row[2],
                "label": row[3]
            }
        )
        i += 1

        if len(data_source) > 100:
            print('+1')
            with db.atomic():
                City.insert_many(data_source).execute()
            data_source = []
            time.sleep(0.02)