import time

import pandas as pd

import gc

import numpy as np

import os

from contextlib import contextmanager

import re

import pandas as pd

import numpy as np

import datetime

data=pd.read_csv('input/feature_0425.csv',low_memory=False)

data = data.rename(columns={"收缩压": "Systolic", "舒张压": "Diastolic", "血清甘油三酯":"triglyceride", "血清高密度脂蛋白":"HDL", "血清低密度脂蛋白":"LDL"})

test_lenth=9538

test=data[-test_lenth:]

train=data[:-test_lenth]

test_vid = test['vid']

target=train.columns[2:7]

cols = list(set(train.columns)- set(target))

vid=data['vid']

temp=data[cols].select_dtypes(include=['object'])

os.environ["OMP_NUM_THREADS"] = "4"

os.environ['CUDA_VISIBLE_DEVICES'] = "0"

@contextmanager

def timer(name):

print(f'[{name}] done in {time.time() - t0:.0f} s')

# with timer ("loading ..."):

# # test = pd.read_csv('test_041717.csv')

# # train = pd.read_csv('train_041717.csv')

# temp = pd.concat([train, test])

cols = temp.columns

mapping ={

}

mapping2 = {

}

with timer ("mapping ..."):

temp = temp.applymap(lambda x : mapping[x] if x in mapping.keys() else x )

# temp = temp.applymap(lambda x : x[1:] if str(x).startswith('>') else x)

# temp = temp.applymap(lambda x : x[1:] if str(x).startswith('<') else x)

# temp = temp.applymap(lambda x : x[1:] if str(x).startswith('﹤') else x)

temp = temp.applymap(lambda x : x[:-1] if str(x).endswith('.') else x)

temp = temp.applymap(lambda x : mapping2[x] if x in mapping2.keys() else x )

obj_list_4 = []

obj_list = []

unit_mapping = ['kpa', 'db/m', '(ng/mL)', '(pmol/L)', '(U/ml)', '%', '＜','kg','(umol/L)']

def unit_transform_s(x):

return y

def unit_transform_x(x):

y = re.sub(r'^<(.*)', r'\1', str(x))

return y

def unit_transform_y(x):

y = re.sub(r'^>(.*)', r'\1', str(x))

return y

def unit_transform_space(x):

return y

cols = list(set(cols) - set(['vid']))

for col in cols:

if (np.array(temp[col]).dtype) == 'object':

try:

temp[col] = temp[col].apply(lambda x : unit_transform_y(x) if str(x).find('>')==0 else x)

except:

print (col)

for col in cols:

if (np.array(temp[col]).dtype) == 'object':

try:

temp[col] = temp[col].apply(lambda x : unit_transform_x(x) if str(x).find('<')==0 else x)

except:

print (col)

for col in cols:

if (np.array(temp[col]).dtype) == 'object':

try:

temp[col] = temp[col].apply(lambda x : unit_transform_space(x) if str(x).find(' ')>0 else x)

except:

print (col)

for col in cols:

if (np.array(temp[col]).dtype) == 'object':

try:

temp[col] = temp[col].apply(lambda x : unit_transform_s(x))

except:

print (col)

def clean(x):

return x

for col in cols:

if (np.array(temp[col]).dtype) == 'object':

try:

temp[col] = temp[col].apply(lambda x:clean(x))

except:

print (col)

def unit_transform_nan(x):

return y

s_cols = ['1321']

for s_col in s_cols:

temp[s_col] = temp[s_col].apply(lambda x : unit_transform_nan(str(x)) if pd.notnull(x) else x)

def unit_transform_yinyang(x):

return y

s_cols = ['2233', '2229']

for s_col in s_cols:

temp[s_col] = temp[s_col].apply(lambda x : unit_transform_yinyang(str(x)) if pd.notnull(x) else x)

def unit_transform_xt(x):

return y

def unit_transform_xt2(x):

return y

def unit_transform_xt3(x):

return y

s_cols = ['424']

for s_col in s_cols:

temp[s_col] = temp[s_col].apply(lambda x : unit_transform_xt(str(x)) if pd.notnull(x) else x)

temp[s_col] = temp[s_col].apply(lambda x : unit_transform_xt2(str(x)) if pd.notnull(x) else x)

temp[s_col] = temp[s_col].apply(lambda x : unit_transform_xt3(str(x)) if pd.notnull(x) else x)

for col in cols:

if (np.array(temp[col]).dtype) == 'object':

obj_list.append(col)

try:

temp[col] = temp[col].apply(lambda x : float(x) )

except:

print (col)

obj_list_4.append(col)

print (pd.unique(temp[col]))

dealcol=[1325, 425 , 437 ,3191 , 547 , 1321, 3203, 2233, 3485 , 30007 , 549, 424 ,459101 , 2229 ,901 ,1322 ,1326 ,3429 ,3430 , 459102 , 3194 ,3198 , 733, 212 , 2302]

dealcol=[str(i) for i in dealcol]

unhealth=temp[dealcol].select_dtypes(include=['object'])

for i in unhealth.columns:

print(i+'*****')

print(unhealth[i].unique())

dropcol=['547', '2302', '733',]

temp=temp.drop(dropcol,axis=1)

num=data.select_dtypes(include=['float64'])

result=pd.concat([temp, num], axis=1)

file_date = datetime.datetime.now().strftime("%m%d%H")

out_file = 'input/feature_'+file_date + '.csv'

if os.path.exists(out_file) == False:

result.to_csv(out_file,index=False,encoding='utf-8')

else:

print ("Have org data")