def kaisaki_create_csv(buscd):
df = kaisaki_dataframe(buscd)
os.makedirs(utils.fullpath('csv'), exist_ok=True)
filename = utils.fullpath('csv/kaisaki_%s.csv' % buscd)
print('%s を作成しています...' % filename)
df.to_csv(filename, index=False, quoting=csv.QUOTE_ALL)
print('%s を作成しました' % filename)
return filename
def hinsyu_create_csv():
df = hinsyu_dataframe()
os.makedirs(utils.fullpath('csv'), exist_ok=True)
filename = utils.fullpath('csv/hinsyu.csv')
print('%s を作成しています...' % filename)
df.to_csv(filename, index=False, quoting=csv.QUOTE_ALL)
print('%s を作成しました' % filename)
return filename
def zaikozan_create_csv(buscd):
buscd = daiei.DAIKYO_BUSCD
df = zaikozan_dataframe(buscd)
os.makedirs(utils.fullpath('csv'), exist_ok=True)
filename = utils.fullpath('csv/zaikozan_%s.csv' % buscd)
print('%s を作成しています...' % filename)
df.to_csv(filename, index=False, quoting=csv.QUOTE_ALL)
print('%s を作成しました' % filename)
return filename
def titles_list(self,
loc=fullpath("data/movies.txt"),
assume_watched=False):
vals = []
if os.path.isfile(fullpath(loc)):
with open(fullpath(loc), mode="r") as f:
readr = csv.DictReader(
f,
delimiter="\t",
fieldnames=["title", "year", "rank", "watched"])
next(readr)
for row in readr:
vals.append(row)
elif os.path.isdir(fullpath(loc)):
ddir = os.walk(fullpath(loc))
files = next(ddir)[2]
for f in files:
name, ext = os.path.splitext(f)
if ext in [
".mkv", ".mp4", ".avi", ".divx", ".mov", ".mpg",
".mpeg"
]:
d = {}
if name.find(")") > -1:
ttl, yr = name.split("(")
yr = yr.replace(")", "")
else:
ttl, yr = name, ""
if assume_watched is False:
d.update({
"title": ttl,
"year": yr,
"rank": -1,
"watched": None
})
else:
d.update({
"title": ttl,
"year": yr,
"rank": -1,
"watched": 1
})
vals.append(d)
else:
pass
else:
raise ValueError("Cannot render titles list from loc")
return vals
def urikaktrn_create_csv(bscd, first, last=None):
if last is None:
last = first
df = urikaktrn_dataframe(buscd, first, last)
os.makedirs(utils.fullpath('csv'), exist_ok=True)
filename = utils.fullpath('csv/urikaktrn_%s_%s_%s.csv' % (
buscd,
first,
last,
))
print('%s を作成しましています...' % filename)
df.to_csv(filename, index=False, quoting=csv.QUOTE_ALL)
print('%s を作成しました' % filename)
def dataframe():
CSV_ROOT = settings.CSV_ROOT
filename = fullpath(CSV_ROOT, 'AITSAKMST_U.csv')
df = pd.read_csv(filename, low_memory=False, dtype=str).fillna('')
return df
def dataframe():
CSV_ROOT = settings.CSV_ROOT
filename = fullpath(CSV_ROOT, 'GZAIKOZAN.csv')
df = pd.read_csv(filename, low_memory=False).fillna('')
return df
def kura_create_csv():
df = kura_dataframe()
filename = utils.fullpath('csv/kura.csv')
print('%s を作成しています...' % filename)
df.to_csv(filename, index=False, quoting=csv.QUOTE_ALL)
print('%s を作成しました' % filename)
return filename
def daiei_tran_tablelist_create_csv(excel, sheets):
for en, ja in sheets:
filename = fullpath('csv/daiei_tran_tablelist_%s.csv' % en)
daiei_tran_tablelist_dataframe(excel, ja)
print('%sを作成しています...' % (filename))
df = daiei_tran_tablelist_dataframe(excel, ja)
df.to_csv(filename, index=False, quoting=csv.QUOTE_ALL)
print('%sを作成しました' % (filename))
def table_list():
# glob files
dirname = CSV_ROOT
filter = '{}{}{}'.format(dirname, os.sep, DESCRIPTION_FILTER)
filenames = glob(filter)
# parse info from filename
data = [table_info(fullpath(filename).name) for filename in filenames]
return data
def createEntryList(database, camel): df = pd.read_html(entl_url(camel))[1] df = firstRowIsColumn(df) entl = df filename = fullpath(root_dir, database, 'entl.csv') os.makedirs(filename.parent, exist_ok=True) entl.to_csv(filename) return entl
def urisaki_create_csv(buscd=None):
df = urisaki_dataframe(buscd)
dirname = 'csv'
os.makedirs(dirname, exist_ok=True)
filename = utils.fullpath(
'csv/urisaki.csv' if buscd is None else 'csv/urisaki_%s.csv' % buscd)
print('%s を作成しています...' % filename)
df.to_csv(filename, index=False, quoting=csv.QUOTE_ALL)
print('%s を作成しました' % filename)
def add_stool(args):
today = datetime.today().strftime('%Y-%m-%d')
with open(utils.fullpath('weight_stool.csv'), 'r+') as f:
lines = f.readlines()
last_day, last_weight, count = lines[-1].split(',')
if last_day == today:
update = f'{today},{last_weight},{int(count)+1}'
lines[-1] = update
f.seek(0, os.SEEK_SET)
f.writelines(lines)
else:
f.write(f'{today},,1')
def tran_create_filename(tablaname, buscd, ymd):
ymd1 = ymd + relativedelta(day=1)
ymd2 = ymd + relativedelta(day=31)
filename = '{}_{}_{:%Y-%m-%d}_{:%Y-%m-%d}.csv'.format(
tablename, buscd, ymd1, ymd2)
filename = fullpath(CSV_ROOT, filename)
if filename.exists() == False:
raise Exception('{}: ファイルが見つかりません'.format(filename))
return filename
def get_unlabeled(self, pseudo_count):
in_theaters = fandango.Theater()
if os.path.isfile(fullpath("data/fandango.csv")):
lst = [
"ID", "actors", "boxoffice_todate", "countries", "directors",
"dvd_release", "genres", "imdbID", "imdbRating", "imdbVotes",
"langs", "mpaa_rating", "plot", "poster_url", "prodco",
"release_date", "release_year", "runtime", "title", "writers",
"watched"
]
unlabeled = read_csv(fullpath("data/fandango.csv"), index_col=0)
else:
lst = in_theaters.listings()
in_theaters.save(data=lst)
unlabeled = lst
unlab = self._pseudo_cases(data=unlabeled.to_dict(orient="records"),
n=pseudo_count,
classx=0)
return unlab
def show(args):
with open(utils.fullpath('weight_stool.csv'), 'r') as f:
print('stool:')
for line in f.readlines()[-args.days:]:
print(line, end='')
df = utils.load_feedings()
by_date = pd.pivot_table(df,
values='Vol',
index=df.index.date,
columns=['BM'],
aggfunc=np.sum,
fill_value=0)
print(by_date.tail(args.days))
def table_describe_dataframe(tablename):
dirname = CSV_ROOT
filter = describe_filename_filter(tablename)
filter = '{}{}{}'.format(dirname, os.sep, filter)
filenames = glob(filter)
if len(filenames) == 0:
raise Exception('{}: パターンのファイルが見つかりません'.format(filter))
filename = fullpath(filenames[0])
info = table_info(filename.name)
df = pd.read_csv(filename).fillna('')
return df, info
def createAttributes(database, camel):
entl = createEntryList(database, camel)
tables = pd.read_html(entd_url(camel))
tables = [
firstRowIsColumn(df) for df in tables
]
tables = [
df for df in tables if 'Column name' in df.columns
]
for i, name in enumerate(entl.Name):
df = tables[i]
filename = fullpath(attributes_csv(database, name))
generateCsv(df, filename)
def _get_labeled(self, pseudo_count):
db = Database(fullpath("data/data.db"))
data = Data(mediatype="movie")
titles = data.titles_list()
labeled = []
try:
select_existing = db.select(table="main", fields="all")
except OperationalError:
data.collect(args=data.titles_list(),
addtl_items=["boxoffice", "oscars", "review"])
i = 0
for movie in labeled:
if i == 0:
print("Making Table main")
db.make_table(table="main",
fields=[k for k in movie.keys()])
else:
pass
db.update(row=movie)
db.conn.commit()
i += 1
else:
if len(select_existing) == len(titles):
header = select_existing[0].keys()
for x in select_existing:
labeled.append(dict(map(lambda x, y: (x, y), header, x)))
else:
data.collect(args=data.titles_list(),
addtl_items=["boxoffice", "oscars", "review"])
i = 0
for movie in labeled:
if i == 0:
print("Making Table main")
db.make_table(table="main",
fields=[k for k in movie.keys()])
else:
pass
db.update(row=movie)
db.conn.commit()
i += 1
lab = self._pseudo_cases(data=labeled, n=pseudo_count, classx="0")
return lab
def save(self, dct, to_db=True, to_csv=False):
if to_db is True:
self._mktable()
stmt = """INSERT INTO review_stars(id, title, release_year, observed_stars, measured_stars, diff, perc_err, sentiment, sentiment_score, review_url) VALUES (?,?,?,?,?,?,?,?,?,?)"""
curs.execute(stmt, tuple([v for v in dct.values()]))
if to_csv is True:
filepath = utils.fullpath("../review_stars.csv")
header = ["id", "title", "release_year", "observed_stars",
"measured_stars", "diff", "perc_err", "sentiment",
"sentiment_score", "review_url"]
with open(filepath, mode="a") as f:
writr = DictWriter(f, fieldnames=header)
if os.path.isfile(filepath):
pass
else:
writr.writeheader()
writr.writerow(dct)
def __init__(self, width, height):
"""---------Create Camera--------"""
camera = Camera()
self.camera = camera
"""-------Create Matricies-------"""
model = glm.mat4()
view = camera.getViewMatrix()
projection = glm.perspective(glm.radians(45.0), width/height, 0.1, 100.0)
"""--------------Compile Shaders--------------"""
shader = Shader(fullpath("shaderPC.vs"), fullpath("shaderPC.fs"))
shader.use()
shader.setMatrix("view", view)
shader.setMatrix("projection", projection)
shader.setMatrix("model", model)
# for drawing position and color - data
self.shaderPC = shader
shader = Shader(fullpath("shaderPT.vs"), fullpath("shaderPT.fs"))
shader.use()
shader.setMatrix("view", view)
shader.setMatrix("projection", projection)
shader.setMatrix("model", model)
# for drawing position and texture - data
self.shaderPT = shader
shader = Shader(fullpath("shaderPCT.vs"), fullpath("shaderPCT.fs"))
shader.use()
shader.setMatrix("view", view)
shader.setMatrix("projection", projection)
shader.setMatrix("model", model)
# for drawing position, color and texture - data
self.shaderPCT = shader
shader = Shader(fullpath("GUIshader.vs"), fullpath("GUIshader.fs"))
# for drawing GUI elements
self.GUIshader = shader
def daiei_columnlist_create_csv(filename,
name_cell=name_cell,
page_cell=page_cell,
cols=cols,
nrows=nrows):
match = re.findall(
r'([A-Z][0-9|x]+)[ ]+(?:([^ ]+)[ ]+)?([^ ]+).(?:XLS|xls)[xX]?$',
str(filename))
if len(match) == 0:
raise Exception('ファイル名が不正です "%s"' % filename.name)
match = match[0]
tablename, pagesize = daiei_columnlist_read_name_and_pagesize(
filename, name_cell, page_cell)
all = pd.DataFrame()
tablename = tablename.lower()
for i in range(pagesize):
print('%s の %dページ目を読み込んでいます...' % (filename.name, i + 1))
df = daiei_columnlist_dataframe(filename, i, cols, nrows)
all = pd.concat([all, df], ignore_index=True)
print('%s の %dページ目を読み込みました' % (filename.name, i + 1))
mxx, tn, ja = match
if tn == '':
tn = tablename.upper()
prefix = '%s_%s_%s' % (mxx, tn, ja)
filename = fullpath('csv/_describe_[%s]_(%s).csv' % (prefix, tablename))
print('%s を作成しています...' % filename)
all.to_csv(filename, index=False, quoting=csv.QUOTE_ALL)
print('%s を作成しました' % filename)
print(all)
from fastapi import APIRouter
import pandas as pd
import settings
##############################################
# router
##############################################
router = APIRouter()
@router.get('/')
async def urisaki_dataframe_route(buscd='0281'):
df = urisaki_dataframe(buscd)
return df.to_dict(orient='record')
##############################################
# constants
##############################################
CSV_ROOT = settings.CSV_ROOT
URISAKI_CSV = fullpath(CSV_ROOT, 'urisaki.csv')
###############################################
# dataframe
###############################################
def urisaki_dataframe(buscd):
filename = URISAKI_CSV
if not buscd is None:
name = '{}_{}{}'.format(
filename.stem,
buscd,
filename.suffix,
)
filename = filename.parent / name
if filename.exists() == False:
def __init__(self, sep="|"):
self.sep = sep
self.datadir = fullpath("data/movielens/")
from utils import fullpath
import os
import csv
datadir = fullpath("data/files/movielens/")
filenames = ["u.data","u.genre","u.info","u.item","u.occupation","u.user"]
datafiles = list(map(lambda x: os.path.join(datadir, x), filenames))
class Data(object):
def __init__(self, sep="|"):
self.sep = sep
self.datadir = fullpath("data/movielens/")
def _read_tabfile(self, file):
with open(file, mode="r") as fx:
readx = csv.reader(fx, delimiter=self.sep)
next(readx)
for row in readx:
yield row
def genres_map(self):
filex = os.path.join(self.datadir, "u.genre")
vals = list(next(self._read_tabfile(file=filex)))
return vals
def items(self):
filex = os.path.join(self.datadir, "u.item")
val = next(self._read_tabfile(file=filex))
def users_info(self):
from utils import fullpath
import pandas as pd
import os
# database = 'crm'
# camel = 'CRM'
root_dir = fullpath('../myobexo')
entl_url = lambda camel: 'http://help.myob.com.au/exo/schemas/exo87/{}/entl.html'.format(camel)
entd_url = lambda camel: 'http://help.myob.com.au/exo/schemas/exo87/{}/entd.html'.format(camel)
def attributes_csv(database, name):
return root_dir / database / 'attributes' / '{}.csv'.format(name)
def firstRowIsColumn(df):
df.columns = df.iloc[0]
return df.reindex(df.index.drop(0))
def generateCsv(df, filename):
os.makedirs(filename.parent, exist_ok=True)
df.to_csv(filename)
def createEntryList(database, camel):
df = pd.read_html(entl_url(camel))[1]
df = firstRowIsColumn(df)
entl = df
filename = fullpath(root_dir, database, 'entl.csv')
os.makedirs(filename.parent, exist_ok=True)
from utils import fullpath
from dotenv import load_dotenv
import os
load_dotenv(dotenv_path=fullpath('../.env'))
CSV_ROOT = fullpath('..', os.getenv('CSV_ROOT'))
from utils import fullpath
from fastapi import APIRouter
import pandas as pd
import settings
#########################################
# router
#########################################
router = APIRouter()
@router.get('/')
async def hinsyu_dataframe_route(offset=0, chunk=1000):
df = hinsyu_dataframe(offset, chunk)
return df.to_dict(orient='record')
#########################################
# constants
#########################################
CSV_ROOT = settings.CSV_ROOT
HINSYU_CSV = fullpath(CSV_ROOT, 'hinsyu.csv')
def hinsyu_dataframe(offset, chunk):
filename = HINSYU_CSV
offset = int(offset)
chunk = int(chunk)
df = pd.read_csv(filename, low_memory=False).fillna('')
df = df[offset:offset + chunk]
return df
from utils import fullpath
from datetime import date
from dateutil.parser import parse
from dateutil.relativedelta import relativedelta
import pandas as pd
buscd = '0281'
ymd = parse('2020-02-22')
ymd1 = ymd + relativedelta(day=1)
ymd2 = ymd + relativedelta(day=31)
filename = fullpath('csv/urikaktrn_{}_{}_{}.csv'.format(
buscd,
ymd1.strftime('%Y-%m-%d'),
ymd2.strftime('%Y-%m-%d'),
))
if filename.exists() == False:
raise Exception('%s が見つかりません' % filename)
df = pd.read_csv(filename)
def add_feed(args):
now = datetime.now().strftime('%Y-%m-%dT%H:%M')
with open(utils.fullpath('feed.csv'), 'a') as f:
f.write(f'{now},{args.volume},{int(not args.formula)}')
