def kaisaki_create_csv(buscd): df = kaisaki_dataframe(buscd) os.makedirs(utils.fullpath('csv'), exist_ok=True) filename = utils.fullpath('csv/kaisaki_%s.csv' % buscd) print('%s を作成しています...' % filename) df.to_csv(filename, index=False, quoting=csv.QUOTE_ALL) print('%s を作成しました' % filename) return filename
def hinsyu_create_csv(): df = hinsyu_dataframe() os.makedirs(utils.fullpath('csv'), exist_ok=True) filename = utils.fullpath('csv/hinsyu.csv') print('%s を作成しています...' % filename) df.to_csv(filename, index=False, quoting=csv.QUOTE_ALL) print('%s を作成しました' % filename) return filename
def zaikozan_create_csv(buscd): buscd = daiei.DAIKYO_BUSCD df = zaikozan_dataframe(buscd) os.makedirs(utils.fullpath('csv'), exist_ok=True) filename = utils.fullpath('csv/zaikozan_%s.csv' % buscd) print('%s を作成しています...' % filename) df.to_csv(filename, index=False, quoting=csv.QUOTE_ALL) print('%s を作成しました' % filename) return filename
def titles_list(self, loc=fullpath("data/movies.txt"), assume_watched=False): vals = [] if os.path.isfile(fullpath(loc)): with open(fullpath(loc), mode="r") as f: readr = csv.DictReader( f, delimiter="\t", fieldnames=["title", "year", "rank", "watched"]) next(readr) for row in readr: vals.append(row) elif os.path.isdir(fullpath(loc)): ddir = os.walk(fullpath(loc)) files = next(ddir)[2] for f in files: name, ext = os.path.splitext(f) if ext in [ ".mkv", ".mp4", ".avi", ".divx", ".mov", ".mpg", ".mpeg" ]: d = {} if name.find(")") > -1: ttl, yr = name.split("(") yr = yr.replace(")", "") else: ttl, yr = name, "" if assume_watched is False: d.update({ "title": ttl, "year": yr, "rank": -1, "watched": None }) else: d.update({ "title": ttl, "year": yr, "rank": -1, "watched": 1 }) vals.append(d) else: pass else: raise ValueError("Cannot render titles list from loc") return vals
def urikaktrn_create_csv(bscd, first, last=None): if last is None: last = first df = urikaktrn_dataframe(buscd, first, last) os.makedirs(utils.fullpath('csv'), exist_ok=True) filename = utils.fullpath('csv/urikaktrn_%s_%s_%s.csv' % ( buscd, first, last, )) print('%s を作成しましています...' % filename) df.to_csv(filename, index=False, quoting=csv.QUOTE_ALL) print('%s を作成しました' % filename)
def dataframe(): CSV_ROOT = settings.CSV_ROOT filename = fullpath(CSV_ROOT, 'AITSAKMST_U.csv') df = pd.read_csv(filename, low_memory=False, dtype=str).fillna('') return df
def dataframe(): CSV_ROOT = settings.CSV_ROOT filename = fullpath(CSV_ROOT, 'GZAIKOZAN.csv') df = pd.read_csv(filename, low_memory=False).fillna('') return df
def kura_create_csv(): df = kura_dataframe() filename = utils.fullpath('csv/kura.csv') print('%s を作成しています...' % filename) df.to_csv(filename, index=False, quoting=csv.QUOTE_ALL) print('%s を作成しました' % filename) return filename
def daiei_tran_tablelist_create_csv(excel, sheets): for en, ja in sheets: filename = fullpath('csv/daiei_tran_tablelist_%s.csv' % en) daiei_tran_tablelist_dataframe(excel, ja) print('%sを作成しています...' % (filename)) df = daiei_tran_tablelist_dataframe(excel, ja) df.to_csv(filename, index=False, quoting=csv.QUOTE_ALL) print('%sを作成しました' % (filename))
def table_list(): # glob files dirname = CSV_ROOT filter = '{}{}{}'.format(dirname, os.sep, DESCRIPTION_FILTER) filenames = glob(filter) # parse info from filename data = [table_info(fullpath(filename).name) for filename in filenames] return data
def createEntryList(database, camel): df = pd.read_html(entl_url(camel))[1] df = firstRowIsColumn(df) entl = df filename = fullpath(root_dir, database, 'entl.csv') os.makedirs(filename.parent, exist_ok=True) entl.to_csv(filename) return entl
def urisaki_create_csv(buscd=None): df = urisaki_dataframe(buscd) dirname = 'csv' os.makedirs(dirname, exist_ok=True) filename = utils.fullpath( 'csv/urisaki.csv' if buscd is None else 'csv/urisaki_%s.csv' % buscd) print('%s を作成しています...' % filename) df.to_csv(filename, index=False, quoting=csv.QUOTE_ALL) print('%s を作成しました' % filename)
def add_stool(args): today = datetime.today().strftime('%Y-%m-%d') with open(utils.fullpath('weight_stool.csv'), 'r+') as f: lines = f.readlines() last_day, last_weight, count = lines[-1].split(',') if last_day == today: update = f'{today},{last_weight},{int(count)+1}' lines[-1] = update f.seek(0, os.SEEK_SET) f.writelines(lines) else: f.write(f'{today},,1')
def tran_create_filename(tablaname, buscd, ymd): ymd1 = ymd + relativedelta(day=1) ymd2 = ymd + relativedelta(day=31) filename = '{}_{}_{:%Y-%m-%d}_{:%Y-%m-%d}.csv'.format( tablename, buscd, ymd1, ymd2) filename = fullpath(CSV_ROOT, filename) if filename.exists() == False: raise Exception('{}: ファイルが見つかりません'.format(filename)) return filename
def get_unlabeled(self, pseudo_count): in_theaters = fandango.Theater() if os.path.isfile(fullpath("data/fandango.csv")): lst = [ "ID", "actors", "boxoffice_todate", "countries", "directors", "dvd_release", "genres", "imdbID", "imdbRating", "imdbVotes", "langs", "mpaa_rating", "plot", "poster_url", "prodco", "release_date", "release_year", "runtime", "title", "writers", "watched" ] unlabeled = read_csv(fullpath("data/fandango.csv"), index_col=0) else: lst = in_theaters.listings() in_theaters.save(data=lst) unlabeled = lst unlab = self._pseudo_cases(data=unlabeled.to_dict(orient="records"), n=pseudo_count, classx=0) return unlab
def show(args): with open(utils.fullpath('weight_stool.csv'), 'r') as f: print('stool:') for line in f.readlines()[-args.days:]: print(line, end='') df = utils.load_feedings() by_date = pd.pivot_table(df, values='Vol', index=df.index.date, columns=['BM'], aggfunc=np.sum, fill_value=0) print(by_date.tail(args.days))
def table_describe_dataframe(tablename): dirname = CSV_ROOT filter = describe_filename_filter(tablename) filter = '{}{}{}'.format(dirname, os.sep, filter) filenames = glob(filter) if len(filenames) == 0: raise Exception('{}: パターンのファイルが見つかりません'.format(filter)) filename = fullpath(filenames[0]) info = table_info(filename.name) df = pd.read_csv(filename).fillna('') return df, info
def createAttributes(database, camel): entl = createEntryList(database, camel) tables = pd.read_html(entd_url(camel)) tables = [ firstRowIsColumn(df) for df in tables ] tables = [ df for df in tables if 'Column name' in df.columns ] for i, name in enumerate(entl.Name): df = tables[i] filename = fullpath(attributes_csv(database, name)) generateCsv(df, filename)
def _get_labeled(self, pseudo_count): db = Database(fullpath("data/data.db")) data = Data(mediatype="movie") titles = data.titles_list() labeled = [] try: select_existing = db.select(table="main", fields="all") except OperationalError: data.collect(args=data.titles_list(), addtl_items=["boxoffice", "oscars", "review"]) i = 0 for movie in labeled: if i == 0: print("Making Table main") db.make_table(table="main", fields=[k for k in movie.keys()]) else: pass db.update(row=movie) db.conn.commit() i += 1 else: if len(select_existing) == len(titles): header = select_existing[0].keys() for x in select_existing: labeled.append(dict(map(lambda x, y: (x, y), header, x))) else: data.collect(args=data.titles_list(), addtl_items=["boxoffice", "oscars", "review"]) i = 0 for movie in labeled: if i == 0: print("Making Table main") db.make_table(table="main", fields=[k for k in movie.keys()]) else: pass db.update(row=movie) db.conn.commit() i += 1 lab = self._pseudo_cases(data=labeled, n=pseudo_count, classx="0") return lab
def save(self, dct, to_db=True, to_csv=False): if to_db is True: self._mktable() stmt = """INSERT INTO review_stars(id, title, release_year, observed_stars, measured_stars, diff, perc_err, sentiment, sentiment_score, review_url) VALUES (?,?,?,?,?,?,?,?,?,?)""" curs.execute(stmt, tuple([v for v in dct.values()])) if to_csv is True: filepath = utils.fullpath("../review_stars.csv") header = ["id", "title", "release_year", "observed_stars", "measured_stars", "diff", "perc_err", "sentiment", "sentiment_score", "review_url"] with open(filepath, mode="a") as f: writr = DictWriter(f, fieldnames=header) if os.path.isfile(filepath): pass else: writr.writeheader() writr.writerow(dct)
def __init__(self, width, height): """---------Create Camera--------""" camera = Camera() self.camera = camera """-------Create Matricies-------""" model = glm.mat4() view = camera.getViewMatrix() projection = glm.perspective(glm.radians(45.0), width/height, 0.1, 100.0) """--------------Compile Shaders--------------""" shader = Shader(fullpath("shaderPC.vs"), fullpath("shaderPC.fs")) shader.use() shader.setMatrix("view", view) shader.setMatrix("projection", projection) shader.setMatrix("model", model) # for drawing position and color - data self.shaderPC = shader shader = Shader(fullpath("shaderPT.vs"), fullpath("shaderPT.fs")) shader.use() shader.setMatrix("view", view) shader.setMatrix("projection", projection) shader.setMatrix("model", model) # for drawing position and texture - data self.shaderPT = shader shader = Shader(fullpath("shaderPCT.vs"), fullpath("shaderPCT.fs")) shader.use() shader.setMatrix("view", view) shader.setMatrix("projection", projection) shader.setMatrix("model", model) # for drawing position, color and texture - data self.shaderPCT = shader shader = Shader(fullpath("GUIshader.vs"), fullpath("GUIshader.fs")) # for drawing GUI elements self.GUIshader = shader
def daiei_columnlist_create_csv(filename, name_cell=name_cell, page_cell=page_cell, cols=cols, nrows=nrows): match = re.findall( r'([A-Z][0-9|x]+)[ ]+(?:([^ ]+)[ ]+)?([^ ]+).(?:XLS|xls)[xX]?$', str(filename)) if len(match) == 0: raise Exception('ファイル名が不正です "%s"' % filename.name) match = match[0] tablename, pagesize = daiei_columnlist_read_name_and_pagesize( filename, name_cell, page_cell) all = pd.DataFrame() tablename = tablename.lower() for i in range(pagesize): print('%s の %dページ目を読み込んでいます...' % (filename.name, i + 1)) df = daiei_columnlist_dataframe(filename, i, cols, nrows) all = pd.concat([all, df], ignore_index=True) print('%s の %dページ目を読み込みました' % (filename.name, i + 1)) mxx, tn, ja = match if tn == '': tn = tablename.upper() prefix = '%s_%s_%s' % (mxx, tn, ja) filename = fullpath('csv/_describe_[%s]_(%s).csv' % (prefix, tablename)) print('%s を作成しています...' % filename) all.to_csv(filename, index=False, quoting=csv.QUOTE_ALL) print('%s を作成しました' % filename) print(all)
from fastapi import APIRouter import pandas as pd import settings ############################################## # router ############################################## router = APIRouter() @router.get('/') async def urisaki_dataframe_route(buscd='0281'): df = urisaki_dataframe(buscd) return df.to_dict(orient='record') ############################################## # constants ############################################## CSV_ROOT = settings.CSV_ROOT URISAKI_CSV = fullpath(CSV_ROOT, 'urisaki.csv') ############################################### # dataframe ############################################### def urisaki_dataframe(buscd): filename = URISAKI_CSV if not buscd is None: name = '{}_{}{}'.format( filename.stem, buscd, filename.suffix, ) filename = filename.parent / name if filename.exists() == False:
def __init__(self, sep="|"): self.sep = sep self.datadir = fullpath("data/movielens/")
from utils import fullpath import os import csv datadir = fullpath("data/files/movielens/") filenames = ["u.data","u.genre","u.info","u.item","u.occupation","u.user"] datafiles = list(map(lambda x: os.path.join(datadir, x), filenames)) class Data(object): def __init__(self, sep="|"): self.sep = sep self.datadir = fullpath("data/movielens/") def _read_tabfile(self, file): with open(file, mode="r") as fx: readx = csv.reader(fx, delimiter=self.sep) next(readx) for row in readx: yield row def genres_map(self): filex = os.path.join(self.datadir, "u.genre") vals = list(next(self._read_tabfile(file=filex))) return vals def items(self): filex = os.path.join(self.datadir, "u.item") val = next(self._read_tabfile(file=filex)) def users_info(self):
from utils import fullpath import pandas as pd import os # database = 'crm' # camel = 'CRM' root_dir = fullpath('../myobexo') entl_url = lambda camel: 'http://help.myob.com.au/exo/schemas/exo87/{}/entl.html'.format(camel) entd_url = lambda camel: 'http://help.myob.com.au/exo/schemas/exo87/{}/entd.html'.format(camel) def attributes_csv(database, name): return root_dir / database / 'attributes' / '{}.csv'.format(name) def firstRowIsColumn(df): df.columns = df.iloc[0] return df.reindex(df.index.drop(0)) def generateCsv(df, filename): os.makedirs(filename.parent, exist_ok=True) df.to_csv(filename) def createEntryList(database, camel): df = pd.read_html(entl_url(camel))[1] df = firstRowIsColumn(df) entl = df filename = fullpath(root_dir, database, 'entl.csv') os.makedirs(filename.parent, exist_ok=True)
from utils import fullpath from dotenv import load_dotenv import os load_dotenv(dotenv_path=fullpath('../.env')) CSV_ROOT = fullpath('..', os.getenv('CSV_ROOT'))
from utils import fullpath from fastapi import APIRouter import pandas as pd import settings ######################################### # router ######################################### router = APIRouter() @router.get('/') async def hinsyu_dataframe_route(offset=0, chunk=1000): df = hinsyu_dataframe(offset, chunk) return df.to_dict(orient='record') ######################################### # constants ######################################### CSV_ROOT = settings.CSV_ROOT HINSYU_CSV = fullpath(CSV_ROOT, 'hinsyu.csv') def hinsyu_dataframe(offset, chunk): filename = HINSYU_CSV offset = int(offset) chunk = int(chunk) df = pd.read_csv(filename, low_memory=False).fillna('') df = df[offset:offset + chunk] return df
from utils import fullpath from datetime import date from dateutil.parser import parse from dateutil.relativedelta import relativedelta import pandas as pd buscd = '0281' ymd = parse('2020-02-22') ymd1 = ymd + relativedelta(day=1) ymd2 = ymd + relativedelta(day=31) filename = fullpath('csv/urikaktrn_{}_{}_{}.csv'.format( buscd, ymd1.strftime('%Y-%m-%d'), ymd2.strftime('%Y-%m-%d'), )) if filename.exists() == False: raise Exception('%s が見つかりません' % filename) df = pd.read_csv(filename)
def add_feed(args): now = datetime.now().strftime('%Y-%m-%dT%H:%M') with open(utils.fullpath('feed.csv'), 'a') as f: f.write(f'{now},{args.volume},{int(not args.formula)}')