def init(self,
target,
file_name,
file_size=100 * 1024 * 1024,
max_file_count=-1,
multiprocess=False):
try:
if multiprocess:
lock = multiprocessing.RLock()
else:
lock = threading.RLock()
if not os.path.exists(target):
os.makedirs(target)
if not os.path.exists(target):
return False
self.__process = Writer(lock, target, file_name + '.process.log',
file_size, max_file_count)
if not self.__process.open():
return False
self.__report = Writer(lock, target, file_name + '.report.log',
file_size, max_file_count)
if not self.__report.open():
return False
if not SysLog.open():
return False
return True
except:
print traceback.format_exc()
def __init__(self):
self.i2c = I2C(scl=Pin('SCL'), sda=Pin('SDA'))
self.screen = SSD1306_I2C(128, 64, self.i2c)
self.wrt = Writer(self.screen, freesans20, False),\
Writer(self.screen, font10, False)
self._msg_ = 'Iniciando'
self._data_ = None
def write_to(self, driver: SH1106_I2C):
from model import OP_MODE_OFF, STATE_HEAT
is_heating = (self.data.state == STATE_HEAT)
wri_t = Writer(driver, freesans40, verbose=False)
wri_t.set_clip(False, False, False) # Char wrap
Writer.set_textpos(driver, 16 + ROW_OFFSET, 26)
if is_heating:
driver.fill_rect(0, 14 + ROW_OFFSET, driver.width - 10, wri_t.height(), 1)
wri_t.printstring(str(int(self.data.current_temperature)) + ".", invert=is_heating)
wri_t_s = Writer(driver, freesans23, verbose=False)
wri_t_s.set_clip(False, False, False) # Char wrap
Writer.set_textpos(driver, 29 + ROW_OFFSET, 85)
wri_t_s.printstring(str(self.data.current_temperature)[-1:], invert=is_heating)
if is_heating:
driver.fill_rect(0, 52 + ROW_OFFSET, driver.width, 4, 0)
driver.text("H", driver.width - 8, 16 + ROW_OFFSET)
driver.text("E", driver.width - 8, 16 + 9 + ROW_OFFSET)
driver.text("A", driver.width - 8, 16 + 9 * 2 + ROW_OFFSET)
driver.text("T", driver.width - 8, 16 + 9 * 3 + ROW_OFFSET)
driver.text("{0:.1f}%RH".format(self.data.sensor_sample.h), 0, ROW_OFFSET)
pressure_str = "{0:.1f}kPa".format(self.data.sensor_sample.p / 10)
driver.text(pressure_str, driver.width - len(pressure_str) * 8, 0 + ROW_OFFSET)
driver.text("room", driver.height - 16, 56 + ROW_OFFSET)
if self.data.operation_mode == OP_MODE_OFF:
driver.text("OFF", driver.width - 24, 20 + ROW_OFFSET)
def dual(use_spi=False, soft=True):
ssd0 = setup(False, soft) # I2C display
ssd1 = setup(True, False) # SPI instance
Writer.set_textpos(ssd0, 0, 0) # In case previous tests have altered it
wri0 = Writer(ssd0, small, verbose=False)
wri0.set_clip(False, False, False)
Writer.set_textpos(ssd1, 0, 0) # In case previous tests have altered it
wri1 = Writer(ssd1, small, verbose=False)
wri1.set_clip(False, False, False)
nfields = []
dy = small.height() + 6
col = 15
for n, wri in enumerate((wri0, wri1)):
nfields.append([])
y = 2
for txt in ('X:', 'Y:', 'Z:'):
Label(wri, y, 0, txt)
nfields[n].append(Label(wri, y, col, wri.stringlen('99.99'), True))
y += dy
for _ in range(10):
for n, wri in enumerate((wri0, wri1)):
for field in nfields[n]:
value = int.from_bytes(uos.urandom(3), 'little') / 167772
field.value('{:5.2f}'.format(value))
wri.device.show()
utime.sleep(1)
for wri in (wri0, wri1):
Label(wri, 0, 64, ' DONE ', True)
wri.device.show()
def total_content(Syear, Smonth, Sday, Fyear, Fmonth, Fday, unique_number):
length = 0
first = ""
start_lst = list()
finish_lst = list()
with open(
f"{PATH}/{Syear}{Smonth}{Sday}_{Fyear}{Fmonth}{Fday}_TStatistic.csv",
newline='') as csvfile:
reader = csv.reader(csvfile)
for i, line in enumerate(reader):
length += 1
if first == "":
first = line[0]
start_lst.append(i)
else:
if first != line[0]:
first = line[0]
start_lst.append(i)
finish_lst.append(i)
finish_lst.append(length)
final_lst = list()
for i in range(len(start_lst)):
dif = finish_lst[i] - start_lst[i]
mine = dif // 10
left = dif % 10
start = start_lst[i] + mine * unique_number
if left > unique_number:
start += unique_number
else:
start += left
if left > unique_number:
mine += 1
if mine != 0:
final_lst.append((start, start + mine))
setter = Setter(Syear, Smonth, Sday, Fyear, Fmonth, Fday, unique_number)
with Pool(processes=50) as pool:
a = pool.map(setter.multi_wrapper, final_lst)
site_lst = list()
for i in a:
try:
site = i[0][1].replace('/', '')
site_lst.append(site)
success_writer = Writer(
f'{Syear}{Smonth}{Sday}_{Fyear}{Fmonth}{Fday}_{site}url{unique_number}'
)
for j in i:
success_writer.write_line(j)
success_writer.close()
except:
print(f"Fail {i}")
writer = Writer(
f'{Syear}{Smonth}{Sday}_{Fyear}{Fmonth}{Fday}_check{unique_number}')
writer.close()
return site_lst
def split(Syear, Smonth, Sday, Fyear, Fmonth, Fday, threshold, setting):
total_lst = list()
target_lst = dict()
startTime = datetime.now()
if setting:
total_dict = dict()
for i in range(0, 10):
with open(f"{PATH}/setting/setting{i}.json", "r",
encoding='utf-8') as csvfile:
tmp_dict = json.load(csvfile)
if i == 0:
total_dict = tmp_dict
else:
for j in tmp_dict['application'].keys():
for k in tmp_dict['application'][j].keys():
total_dict['application'][j][k] = tmp_dict[
'application'][j][k]
os.remove(f"{PATH}/setting/setting{i}.json")
with open(f"{PATH}/setting/setting.json", 'w', encoding='utf-8') as f:
json.dump(total_dict, f, ensure_ascii=False, indent=2)
del total_dict
for i in range(0, 10):
with open(
f"{PATH}/{Syear}{Smonth}{Sday}_{Fyear}{Fmonth}{Fday}_Statistic{i}.csv",
newline='') as csvfile:
reader = csv.reader(csvfile)
for line in reader:
if (int(line[2]) >= int(threshold)) and (line[1]
not in except_lst):
site = line[0]
if site in target_lst:
key_lst = target_lst[site]
key_lst.append(line)
target_lst[site] = key_lst
else:
target_lst[site] = [line]
total_lst.append(line)
os.remove(
f"{PATH}/{Syear}{Smonth}{Sday}_{Fyear}{Fmonth}{Fday}_Statistic{i}.csv"
)
writer = Writer(f'{Syear}{Smonth}{Sday}_{Fyear}{Fmonth}{Fday}_Statistic')
for i in total_lst:
writer.write_line(i)
writer.close()
writer = Writer(f'{Syear}{Smonth}{Sday}_{Fyear}{Fmonth}{Fday}_TStatistic')
for i in target_lst:
for j in target_lst[i]:
writer.write_line(j)
writer.close()
endTime = datetime.now()
def fonts(use_spi=False, soft=True):
ssd = setup(use_spi, soft) # Create a display instance
Writer.set_textpos(ssd, 0, 0) # In case previous tests have altered it
wri = Writer(ssd, freesans20, verbose=False)
wri.set_clip(False, False, False) # Char wrap
wri_f = Writer(ssd, small, verbose=False)
wri_f.set_clip(False, False, False) # Char wrap
wri_f.printstring('Sunday\n')
wri.printstring('12 Aug 2018\n')
wri.printstring('10.30am')
ssd.show()
def parseUrl(self, url, xpath, type):
headers = {
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.90 Safari/537.36'
}
try:
# Retrying for failed requests
for i in range(20):
response = requests.get(url, headers=headers, verify=False)
if response.status_code == 200:
doc = html.fromstring(response.content)
XPATH_NAME = xpath
RAW_NAME = doc.xpath(XPATH_NAME)
if type == 'body':
print("\r\n")
print('doin body')
LANGUAGE = doc.xpath('//html/@lang')
LANGUAGE = LANGUAGE[0]
BODY = ' '.join(
''.join(RAW_NAME).split()) if RAW_NAME else None
data = BODY
writer = Writer()
ts = time()
timestamp = datetime.fromtimestamp(ts).strftime(
'%Y-%m-%d %H:%M:%S')
if data and LANGUAGE == 'de':
writer.insertBody(data, url, timestamp)
# print('weiter')
# print(url)
# # print(type(url))
# print(timestamp)
# # print(type(timestamp))
# print(LANGUAGE)
# print(type(LANGUAGE))
writer.updateLink(url, timestamp, LANGUAGE)
elif type == 'links':
print('doin links')
print("\r\n")
if RAW_NAME:
writer = Writer()
for link in RAW_NAME:
ts = time()
timestamp = datetime.fromtimestamp(
ts).strftime('%Y-%m-%d %H:%M:%S')
writer.insertLink(link, url, timestamp)
break
elif response.status_code == 404:
print('got status code 404')
break
except Exception as e:
print(e)
def dumpResult(self):
# dump trace data
writer = Writer(self.generateTraceFileName())
self.resetTraceCursor()
writer.overwrite(self.getNextLinesString(800))
while (self.getLineCursorPos() < self.getTotalLineSize()):
writer.writeAtEOF(self.getNextLinesString(800))
# dump analysis result data
writer = Writer(self.generateEvalFileName())
writer.overwrite(self.getAnalysisResult())
def test_string(self):
DATA = ''.join([chr(u) for u, _ in strings.U2C_TABLE.items()])
writer = Writer()
writer.write_string(DATA)
self.assertEqual(Reader(writer.data).read_string(), DATA)
def draw_landscape(self, coin, display_number):
wri_serif = Writer(displays[display_number].driver, LCD_font11p)
Writer.set_clip(True, True)
displays[display_number].driver.fill(0)
displays[display_number].driver.line(0, 25, 128, 26, 1) #horizontal
displays[display_number].driver.line(0, 26, 128, 25, 1) #horizontal
#displays[display_number].driver.fill_rect(0, 0, 30, 10, 1)
wri_serif.printstring(holder[coin].symbol, 3, 1)
if holder[coin].display_currency == "USD":
wri_serif.printstring(
holder[coin].price_usd,
self.calculate_middle_position(holder[coin].price_usd), 30)
else:
wri_serif.printstring(
holder[coin].price_btc,
self.calculate_middle_position(holder[coin].price_btc), 30)
wri_serif.printstring(
holder[coin].percent_change_24h,
self.calculate_corner_position(holder[coin].percent_change_24h), 1)
displays[display_number].driver.show()
def __init__(self):
"""Create with the configuration."""
self._enable_display(config.rst)
i2c = I2C(-1, config.scl, config.sda, freq=config.frequency)
self._oled = ssd1306.SSD1306_I2C(config.width, config.height, i2c)
self._writer = Writer(self._oled, fullheight_bold, verbose=False)
def main(argv=None):
writer = Writer(RESULTS_DIR)
trainer = Trainer(RESULTS_DIR, 'train', writer)
trainer.train(LEARNING_RATE,
EVAL_FREQUENCY,
init_step=None,
restoring_file=RESTORING_FILE)
def __init__(self, args, src_dict, tgt_dict, state_dict=None):
# Book-keeping.
self.args = args
self.word_dict = src_dict
self.args.vocab_size = len(src_dict)
self.tgt_dict = tgt_dict
self.args.tgt_vocab_size = len(tgt_dict)
self.updates = 0
self.use_cuda = False
self.parallel = False
if args.model_type == 'rnn':
self.network = Writer(args, tgt_dict)
else:
raise RuntimeError('Unsupported model: %s' % args.model_type)
# Load saved state
if state_dict:
# Load buffer separately
if 'fixed_embedding' in state_dict:
fixed_embedding = state_dict.pop('fixed_embedding')
self.network.load_state_dict(state_dict)
self.network.register_buffer('fixed_embedding', fixed_embedding)
else:
self.network.load_state_dict(state_dict)
if self.args.ema:
self.ema = ExponentialMovingAverage(0.999)
for name, param in self.network.named_parameters():
if param.requires_grad:
self.ema.register(name, param.data)
def execute(input_folder_name, output_folder_name, file_num, data_hash):
input_name = ('input%0{}d.txt'.format(len(
str(number_of_files)))) % file_num
output_name = ('output%0{}d.txt'.format(len(
str(number_of_files)))) % file_num
rd = reader.Reader(input_folder_name=input_folder_name)
writer = Writer(output_folder_name=output_folder_name)
simplifier = Simplifier()
execution_lt = [
(simplifier.tautoly, "tautoly"),
(simplifier.blocked_clause, "blocked_clause"),
(simplifier.subsumption_elimination, "subsumption_elimination"),
(simplifier.hidden_tautoly, "hidden_tautoly"),
(simplifier.hidden_blocked_clause, "hidden_blocked_clause"),
(simplifier.hidden_subsumption_elimination,
"hidden_subsumption_elimination"),
(simplifier.asymmetric_tautoly, "asymmetric_tautoly"),
(simplifier.asymmetric_blocked_clause, "asymmetric_blocked_clause"),
(simplifier.asymmetric_subsumption_elimination,
"asymmetric_subsumption_elimination"),
(simplifier.explicits, "explicits"),
(simplifier.hiddens, "hiddens"),
(simplifier.asymmetrics, "asymmetrics"),
(simplifier.complete, "Complete"),
]
for function, function_name in execution_lt:
execution(function, rd, writer, input_name, output_name, function_name,
data_hash)
def __init__(self, ml, subreddit, location):
"""Initialize the Markov Chain and writer
ml:
MarkovChain's max_links
subreddit:
The subreddit to use as the text source
location:
"posts"|"comments" - Whether to get the text from the top posts in
the subreddit (faster), or from the children comments of the top
posts in the subreddit (can return more text).
sources:
An array of strings for the MC
"""
rr = RedditReader(subreddit)
if location == "posts":
texts = rr.get_many_post_bodies()
elif location == "comments":
texts = rr.get_many_comment_bodies()
else:
raise TypeError('`location` must be either "posts" or "comments"')
self.mc = MarkovChain(ml)
for text in texts:
self.mc.add_text(text)
self.w = Writer(self.mc)
def crawling(self, category_name):
# Multi Process PID
print(category_name + " PID: " + str(os.getpid()))
writer = Writer(category_name=category_name, date=self.date)
wcsv = writer.get_writer_csv()
wcsv.writerow([
"date", "time", "category", "company", "author", "headline",
"sentence", "content_url", "image_url"
])
# 기사 URL 형식
url = "http://news.naver.com/main/list.nhn?mode=LSD&mid=sec&sid1=" + str(
self.categories.get(category_name)) + "&date="
# start_year년 start_month월 ~ end_year의 end_month 날짜까지 기사를 수집합니다.
day_urls = self.make_news_page_url(url, self.date['year'],
self.date['month'],
self.date['day'])
print(category_name + " Urls are generated")
print("The crawler starts")
with concurrent.futures.ThreadPoolExecutor() as pool:
futureWorkers = []
for URL in day_urls:
futureWorkers.append(
pool.submit(
self.get_page_and_write_row,
category_name,
writer,
URL,
))
for future in concurrent.futures.as_completed(futureWorkers):
print(future.result())
writer.close()
def hello():
reader = Reader()
result = reader.read('./input.txt')
calculator = Calculator(result)
calculatedResult = calculator.getResult()
Writer(calculatedResult)
return calculator.getResultAsJSON()
def __init__(self, config):
"""Weibo类初始化"""
self.config = config
# change cookie from string to dict
if type(self.config['cookie']) == type(u''):
self.config['cookie'] = {
t.strip().split("=")[0]: t.strip().split("=")[1]
for t in self.config['cookie'].split(";")
}
if type(self.config['user_id_list']) == type(u""):
user_id_list = self.config['user_id_list']
if not os.path.isabs(user_id_list):
user_id_list = os.path.split(
os.path.realpath(__file__))[0] + os.sep + user_id_list
self.config['user_id_list'] = user_id_list
with open(self.config['user_id_list'], 'rb') as f:
lines = f.read().splitlines()
lines = [line.decode('utf-8') for line in lines]
self.config['user_id_list'] = [
line.split(' ')[0] for line in lines if
len(line.split(' ')) > 0 and line.split(' ')[0].isdigit()
]
if type(self.config['since_date']) == type(0):
self.config['since_date'] = str(
date.today() - timedelta(self.config['since_date']))
self.validator = Validator(self.config)
self.validator.validate()
self.printer = Printer()
self.writer = Writer(self.config)
self.downloader = Downloader(self.config)
self.parser = Parser(self.config)
def process_butches(self):
scales_obj = Scales()
agregate_obj = Agregator({})
write_obj = Writer(self.outname_restored, self.outname_tofix)
for line in self.query_completion_file:
pair = line.strip().split(';')
if not len(pair) == 2:
continue
scales_obj.weigh_match(pair)
if scales_obj.light_match:
# сбрасывает фрагмент обработанного массива в файлы
if self.n % self.memory_limit == 0:
agregate_obj.agregate_matches(scales_obj.light_match)
write_obj.write_matches(agregate_obj.restored,
agregate_obj.tofix_manually)
agregate_obj = Agregator(self.last_known)
else:
agregate_obj.agregate_matches(scales_obj.light_match)
# если конец фрагмемнта близок, запоминает последние 3 пары
if self.n % self.memory_limit >= self.memory_limit - 3:
self.last_known[scales_obj.light_match.query] = (
scales_obj.light_match.weight,
scales_obj.light_match.init_str,
scales_obj.light_match.complete)
self.n += 1
# делает отчет
if self.n % self.reporting == 0:
print('{:.0f}K lines processed'.format(self.n / 1000))
write_obj.write_matches(agregate_obj.restored,
agregate_obj.tofix_manually)
def wrap(use_spi=False, soft=True):
ssd = setup(use_spi, soft) # Create a display instance
Writer.set_textpos(ssd, 0, 0) # In case previous tests have altered it
wri = Writer(ssd, freesans20, verbose=False)
wri.set_clip(False, False, True) # Word wrap
wri.printstring('the quick brown fox jumps over')
ssd.show()
def main(filename, **kwargs):
writer = Writer(filename, **kwargs)
writer.load_audio()
writer.compute_loudness()
writer.generate_phrases_filter()
writer.generate_chunks()
writer.generate_plot()
def execute(self):
parser = argparse.ArgumentParser(prog=self.prog_name)
parser.add_argument('-o',
dest='outfile',
action='store',
required=True,
help='base dataset name')
parser.add_argument('-m',
'--model',
dest='model',
action='store',
required=True,
help='model to generate')
parser.add_argument('-n',
'--nevents',
dest='nevents',
type=int,
required=True,
help="Total number of events to generate")
parser.add_argument('-s',
'--seed',
dest='seed',
action='store',
help='set seed')
arguments = parser.parse_args(self.argv[1:])
s = Synthesizer('examples', arguments.model, 'en_CA')
writer = Writer(arguments.outfile, s, arguments.nevents)
writer.write()
def handle_standard_io(parser):
parser.read_file(sys.stdin)
#parser.print_file() # what is this for?
parser.build_structure() # build doc from actual file
writer = Writer()
writer.write(parser.node_file)
for line in writer.buffer:
print line.rstrip()
def convert_to_txt(self): converter = Converter(self.file_name+".pdf",self.target_name,self.path) converter.convert() reader = Reader(self.target_name,self.pg_count,self.path) extracted_text = reader.get_text() extracted_text = os.linesep.join([s for s in extracted_text.splitlines() if s]) writer = Writer((self.target_name+".txt"),extracted_text,self.path) writer.write()
def solvepuzzle():
split_puzzle = sys.argv
# parsing the input into sections
puzzle_string = split_puzzle[1]
procedure_name = split_puzzle[2]
o_file = split_puzzle[3]
output_file = open(o_file + ".txt", "a")
# 0 is for the cost of the path at the current state, also the operator
# [puzzle, op, cost]
state = [puzzle_string, 0, 0]
stateList = [state]
global flag
flag = int(split_puzzle[4])
global total_white_no
total_white_no = total_white(puzzle_string)
global writer1
writer1 = Writer(o_file)
if procedure_name == "BK":
print("backtrack")
start_op = ["start"]
solve_puzzle = Backtrack(puzzle_string, output_file, flag, start_op)
solve_puzzle.backtrack(solve_puzzle.past_positions,
solve_puzzle.start_op)
cost = 0
for i in range(len(solve_puzzle.final_path)):
string1 = ''.join(solve_puzzle.final_path[i])
# print(solve_puzzle.final_ops[i], '\t', string1,'\t', cost, file=output_file)
output_file.write(
str(solve_puzzle.final_ops[i]) + '\t' + str(string1) + '\t' +
str(cost))
print(solve_puzzle.final_ops[i], string1, cost)
try:
calc_cost = abs(solve_puzzle.final_path[i].index('E') -
solve_puzzle.final_path[i + 1].index('E'))
if calc_cost > 1:
cost += calc_cost - 1
else:
cost += calc_cost
except IndexError:
pass
print(solve_puzzle.final_ops)
elif procedure_name == "DLS":
tree_search(state)
elif procedure_name == "A":
a(state)
def test(self):
s = Synthesizer('examples', 'SampleModelA', 'en_CA')
print(s.generate())
for _ in range(10):
print(s.generate())
s2 = Synthesizer('examples', 'CustomModel', 'en_CA')
print(s2.generate())
#mywriter = Writer('test', s, 10000)
#mywriter.write()
mywriter2 = Writer('test2', s2, 1000)
mywriter2.write()
assert mywriter2.nevents == mywriter2.eventcount
df = pd.read_csv('test2.0000.csv', header = None)
#print(df)
X = df.loc[:,[3,4]].values
#print(X)
y = df.loc[:,[5]].values
#print(y)
# Fit
# beta = np.array([1, 0.1, 10])
model = sm.OLS(y, X)
results = model.fit()
print(results.summary())
print(*results.params)
y_fit = np.dot(X, (results.params))
#print(y_fit)
# Create a plot of our work, showing both the data and the fit.
fig, ax = plt.subplots(2, 2)
fig.subplots_adjust(hspace=0.3)
ax[0, 0].scatter(df.loc[:,[3]], df.loc[:,[5]])
ax[0, 0].scatter(df.loc[:,[3]], y_fit, color='red')
ax[0, 0].set_xlabel(r'$x$')
ax[0, 0].set_ylabel(r'$f(x)$')
ax[0, 1].scatter(X[:,[1]], y)
#ax[0, 1].scatter(X[:,[1]], y_fit, color='red')
miny=df.loc[:,[5]].min()
maxy=df.loc[:,[5]].max()
print(miny)
print(maxy)
#ax[1,0].hist(y,10,range=[miny,maxy])
#ax[1,0].hist(y_fit,10, range=[miny,maxy],color='red',lw=2)
#ax[0, 0].set_xlabel(r'$x$')
#ax[0, 0].set_ylabel(r'$f(x)$')
fig.savefig('closure.pdf', format='pdf')
def tabs(use_spi=False, soft=True):
ssd = setup(use_spi, soft) # Create a display instance
Writer.set_textpos(ssd, 0, 0) # In case previous tests have altered it
wri = Writer(ssd, fixed, verbose=False)
wri.set_clip(False, False, False) # Char wrap
wri.printstring('1\t2\n')
wri.printstring('111\t22\n')
wri.printstring('1111\t1')
ssd.show()
def __init__(self, url='http://newhouse.hz.fang.com/house/s/',
checkpoint_path='../conf/checkpoint.txt',
writer=Writer('../data/newhouse')):
self.url = url
self.checkpoint_path = checkpoint_path
self.page_num = None
self.curr_page = None # 从1开始数
self.curr_item = None # 从0开始数
self.writer = writer
def test_int32(self):
NUMBERS = [-2147483647, 0, 2147483647]
writer = Writer()
for n in NUMBERS:
writer.write_int32(n)
reader = Reader(writer.data)
for n in NUMBERS:
self.assertEqual(reader.read_int32(), n)
