def prepare(self, langs=None):
LOG.debug("bottle request.headers.keys %s", request.headers.get("Accept-Language", None))
if langs is None:
langs = self.get_language_list()
LOG.debug("web client accept langs: %s", langs)
prepared_key = tuple(langs)
if prepared_key in self.prepared:
trans = self.prepared.get(prepared_key)
if trans:
trans.install(True)
self.app._ = trans.gettext
else:
self.app._ = lambda s: s
return
LOG.debug("setup i18n ...")
try:
trans = gettext.translation(self.domain, self.locale_dir, languages=langs)
trans.install(True)
self.app._ = trans.gettext
self.prepared[prepared_key] = trans
except Exception, e:
LOG.warn('can not install application for language "%s" with locale path as "%s"', langs, self.locale_dir)
LOG.warn(e)
self.app._ = lambda s: s
self.prepared[prepared_key] = None
def set_duty_cycle(self, pin, value):
"""
:param pin:
:param value:
:return:
"""
if pin not in self.output_pins:
log.error("pin %s was not registered as an output" % pin)
return
output_pin = self.output_pins[pin]
if not output_pin['pwm']:
log.error("pwm was not registered at pin %d" % pin)
if value > 100:
log.warning("Given dutycycle (%d) is bigger than 100. Has been set to 100" % value)
value = 100
if value < 0:
log.warning("Given dutycycle (%d) is lower than 0. Has been set to 0" % value)
value = 0
if not output_pin['pwm_startet']:
output_pin['pwm'].start(value)
output_pin['pwm_startet'] = True
else:
output_pin['pwm'].ChangeDutyCycle(value)
output_pin['dutycycle'] = value
log.info("Dutycycle of pin %d has been set to %d" % (pin, value))
def convex_hull(points):
"""Returns the points on the convex hull of points in CCW order."""
# Increasing guesses for the hull size.
for guess in (2**(2**t) for t in range(len(points))):
LOG("Guess", guess)
hulls = []
for i in range(0, len(points), guess):
# LOG(".")
# Split the points into chunks of (roughly) the guess.
chunk = points[i:i + guess]
# Find the corresponding convex hull of these chunks.
hulls.append(graham_scan(chunk))
# Find the extreme point and initialize the list of (hull,point) with it.
hullpt_pairs = [min_hull_pt_pair(hulls)]
# Ensure we stop after no more than "guess" iterations.
for __ in range(guess):
LOG("*")
pair = next_hull_pt_pair(hulls, hullpt_pairs[-1])
if pair == hullpt_pairs[0]:
# Return the points in sequence
LOGN("o")
return [hulls[h][i] for h, i in hullpt_pairs]
hullpt_pairs.append(pair)
LOGN("x")
def _download_video( self ):
try:
# spam log file
LOG( ">>> _download_video(title: %s)" % ( repr( self.g_title ), ), heading=True )
# get filepath and tmp_filepath
tmppath, self.filepath = get_legal_filepath( self.g_title, self.params[ "download" ], self.settings[ "play_mode" ], self.settings[ "download_path" ], self.settings[ "use_title" ], self.settings[ "use_trailer" ] )
# only download if the trailer doesn't exist
if ( not os.path.isfile( self.filepath.encode( "utf-8" ) ) ):
# only need to retrieve video if not in tmp path
if ( not os.path.isfile( tmppath.encode( "utf-8" ) ) ):
# fetch the video
urllib.urlretrieve( self.params[ "download" ], tmppath.encode( "utf-8" ), self._report_hook )
# create the conf file for xbox and copy to final location
ok = self._finalize_download( tmppath )
# if the copy failed raise an error
if ( not ok ): raise
except Exception, e:
# oops, notify user what error occurred
LOG( str( e ), xbmc.LOGERROR )
# filepath is not always released immediately, we may need to try more than one attempt, sleeping between
urllib.urlcleanup()
remove_tries = 3
while remove_tries and os.path.isfile( tmppath ):
try:
os.remove( tmppath.encode( "utf-8" ) )
except:
remove_tries -= 1
xbmc.sleep( 1000 )
pDialog.close()
self.filepath = ""
def _decode_multipart_body(self):
boundary = self._get_boundary()
if not boundary:
LOG.warn("Message detected as multipart but boundary "
"declaration was not found")
return
start_bnd = '\n' + '--' + boundary + '\n'
end_bnd = '\n' + '--' + boundary + '--' + '\n'
self.body = '\n' + self.body # for string matching purpose
try:
start_index = self.body.index(start_bnd) + len(start_bnd)
except ValueError:
LOG.warn("Cannot find boundaries in body, "
"treating as single message")
self._decode_single_body()
return
end_index = self.body.rfind(end_bnd)
if end_index < 0:
end_index = None
content = self.body[start_index:end_index]
parts = content.split(start_bnd)
messages = [MessageParser(self.name, msg_content, self.headers)
for msg_content in parts]
self.body = "\n".join([msg.body for msg in messages])
for msg in messages:
self.email_stats += msg.email_stats
self.html_stats += msg.html_stats
async def predict_sentiment(request):
payload = await request.json()
x = parse_input(payload)
if not x:
return web.Response(
text=
"No or empty input received. Please post your text body as json of the form {'Text': text body string}.",
status=400)
elif x is KeyError:
return web.Response(
text=
"Wrong input. Please post your text body as json of the form {'Text': text body string}.",
status=400)
elif x is ValueError:
return web.Response(
text=
"Wrong input type. Please post your text body as json of the form {'Text': text body string}.",
status=400)
else:
try:
ft = get_features_from_text(x, tfidf_v)
predicted_spam = get_prediction([ft])
return web.json_response(
{"spam probability": round(predicted_spam[0][0], 3)})
except Exception as e:
LOG.error(f"Application errored: {e.__repr__()}")
return web.Response(text="Something has gone very wrong indeed...",
status=500)
def size(self):
size = 0
try:
size = os.path.getsize(self.filepath)
except Exception as e:
LOG.error("Get {0}'s size error".format(self.path))
LOG.exception(e)
return size
def filtered_files(self):
LOG.debug("%s filtered_files", self.__class__.__name__)
ret = list()
for k, v in self.char_table.items():
#LOG.debug("{0} {1}".format(k, v))
if len(v) > 1:
ret.extend(v)
return ret
def filtered_files(self):
LOG.debug("%s filtered_files", self.__class__.__name__)
ret = list()
for k, v in self.char_table.items():
#LOG.debug("{0} {1}".format(k, v))
if len(v) > 1:
ret.extend(v)
return ret
def dup_files(self):
LOG.debug("%s dup_files", self.__class__.__name__)
ret = list()
for k, v in self.char_table.items():
# LOG.debug("{0} {1}".format(k, v))
# LOG.debug(k)
if len(v) > 1:
ret.append(v)
return ret
def dup_files(self):
LOG.debug("%s dup_files", self.__class__.__name__)
ret = list()
for k, v in self.char_table.items():
# LOG.debug("{0} {1}".format(k, v))
# LOG.debug(k)
if len(v) > 1:
ret.append(v)
return ret
def cleanup(self):
"""
"""
log.info("Cleanup")
self._all_output_off()
GPIO.cleanup()
def find(self):
LOG.debug("%s find", self.__class__.__name__)
for _file in self.files:
character = _file.size
entry = self.char_table.get(character)
if entry:
entry.append(_file)
else:
self.char_table[character] = [_file]
self.progress = self.progress + 1
def send_msg(self, sender, target, data):
if self._check_in_black(sender, target):
LOG.info('Packet block %s->%s' % (sender, target))
return
queue = self.queues.get(target)
if queue is not None:
queue.put(data)
else:
raise Exception('Cannot send data')
def detail(id):
blogs = execute_sql('select id, title, created_time, content from blog where id =?', (id,))
if not len(blogs):
raise HTTPError(404, 'Blog does not exist.')
LOG.debug('column created time type: %s', type(blogs[0]['created_time']))
#myapp.set_lang(['jp'])
msg = myapp._('test i18n in py')
LOG.debug('i18n msg: %s', msg)
myapp.set_lang(['ja'])
return {'blog': blogs[0], 'msg':msg, '_': myapp._}
def find(self):
LOG.debug("%s find", self.__class__.__name__)
for _file in self.files:
character = _file.size
entry = self.char_table.get(character)
if entry:
entry.append(_file)
else:
self.char_table[character] = [_file]
self.progress = self.progress + 1
def handle_endtag(self, tag):
if tag in HTML_PARSER_IGNORE_TAGS:
return
if self.tag_stack and tag == self.tag_stack[0]:
del self.tag_stack[0]
else:
LOG.debug("Invalid closing tag at %r", self.getpos())
if tag in self.tag_stack:
idx = self.tag_stack.index(tag)
del self.tag_stack[:idx + 1]
self.stats['errors_count'] += idx + 1
def dump2csv(self, output_csv):
LOG.debug("%s dump2csv", self.__class__.__name__)
rows = list()
for files in self.sorted_dup_files:
data = [utils.size_renderer(files[0].size)]
#data.append(files[0].size)
data.extend([_file.path for _file in files])
rows.append(data)
with open(output_csv, 'wb') as f:
writer = UnicodeCSVWriter(f)
writer.writerows(rows)
def find(self):
LOG.debug("%s find", self.__class__.__name__)
for _file in self.files:
character = _file.character
if character == setting.UNKNOWN_SYMBOL:
continue
entry = self.char_table.get(character)
if entry:
entry.append(_file)
else:
self.char_table[character] = [_file]
self.progress = self.progress + 1
def process_view(self, request, view_func, *view_args, **view_kwargs):
if request.path == '/token/refresh/' and JWT_AUTH_REFRESH_COOKIE in request.COOKIES:
if request.body != b'':
data = json.loads(request.body)
data['refresh'] = request.COOKIES[JWT_AUTH_REFRESH_COOKIE]
request._body = json.dumps(data).encode('utf-8')
else:
LOG.info(
f"\n{L.FAIL} Error in api/users/middleware.py: The incoming request body must be set to an empty object.{L.ENDC}\n"
)
return None
def find(self):
LOG.debug("%s find", self.__class__.__name__)
for _file in self.files:
md5sum = _file.md5sum
if md5sum == setting.UNKNOWN_SYMBOL:
continue
entry = self.char_table.get(md5sum)
if entry:
entry.append(_file)
else:
self.char_table[md5sum] = [_file]
self.progress = self.progress + 1
def find(self):
LOG.debug("%s find", self.__class__.__name__)
for _file in self.files:
md5sum = _file.md5sum
if md5sum == setting.UNKNOWN_SYMBOL:
continue
entry = self.char_table.get(md5sum)
if entry:
entry.append(_file)
else:
self.char_table[md5sum] = [_file]
self.progress = self.progress + 1
def find(self):
LOG.debug("%s find", self.__class__.__name__)
for _file in self.files:
character = _file.character
if character == setting.UNKNOWN_SYMBOL:
continue
entry = self.char_table.get(character)
if entry:
entry.append(_file)
else:
self.char_table[character] = [_file]
self.progress = self.progress + 1
def _decode_body(self):
if self.mime_type and (self.mime_type.startswith('image/') or
self.mime_type.startswith('application/')):
LOG.info("Body marked as image, skipping body")
self.email_stats['attached_images'] += 1
self.body = ""
return
if self.is_multipart:
LOG.debug("Detected multipart/* content-type")
self._decode_multipart_body()
else:
self._decode_single_body()
def parse_input(json_text):
try:
text_body = json_text["Text"]
if type(text_body) != str:
LOG.warning(
f"Wrong user input. User posted us the following input {text_body} of type: {type(text_body)}"
)
return ValueError
return str(text_body).strip()
except KeyError:
LOG.warning(
f"Wrong user input. User posted us the following: {json_text}")
return KeyError
def dump2file(self, output_file):
LOG.debug("%s dump2file", self.__class__.__name__)
if utils.get_python_version() == 3:
fp = codecs.open(output_file, "w", "utf-8")
else:
fp = open(output_file, 'w')
try:
for files in self.sorted_dup_files:
fp.write("================\n")
for _file in files:
size = utils.size_renderer(_file.size)
fp.write("Size: {0}, File: {1}\n".format(size, _file.path))
finally:
fp.close()
def start_find(self):
LOG.debug("start_find button click")
self.disable_all()
# start to find
paths = [self.path_field1.get()]
if (self.path_field2.get()):
paths.append(self.path_field2.get())
if (self.path_field3.get()):
paths.append(self.path_field3.get())
if (self.path_field4.get()):
paths.append(self.path_field4.get())
LOG.debug(paths)
LOG.debug("Full Scan {0}".format(str(self.full_scan.get())))
LOG.debug("Ouput csv {0}".format(str(self.output_csv.get())))
do_it = messagebox.askyesno('',
'It may take several minutes to complete please wait')
if not do_it:
self.enable_all()
return
self.find_complete = False
filters = [
core.algorithm.SizeFilter(),
core.algorithm.CharacterFilter()
]
if (self.full_scan.get()):
filters.append(core.algorithm.FullScanner())
dup_finder = core.dup_finder.DupFinder(paths, filters)
self.status_thread = threading.Thread(target=self.update_status,
args=(dup_finder,))
self.find_thread = threading.Thread(target=self.background_find,
args=(dup_finder,))
self.find_thread.start()
self.status_thread.start()
def analyze_galera(net: Network):
'''Fig2. Analyze of the Galera protocol on the `net` Network.
This function builds a specific `filter` with the CIDR of the
`net` and prints TCP `packets` that are related to the Galera
communications.
'''
LOG.info(f'Listen packet on {ifname(net)}...')
scapy.sniff(
iface=ifname(net),
count=10, # Stop analysis after 10 packets
filter=f'net {net["cidr"]} and tcp and port 4567',
prn=lambda packet: packet.summary())
def start_find(self):
LOG.debug("start_find button click")
self.disable_all()
# start to find
paths = [self.path_field1.get()]
if (self.path_field2.get()):
paths.append(self.path_field2.get())
if (self.path_field3.get()):
paths.append(self.path_field3.get())
if (self.path_field4.get()):
paths.append(self.path_field4.get())
LOG.debug(paths)
LOG.debug("Full Scan {0}".format(str(self.full_scan.get())))
LOG.debug("Ouput csv {0}".format(str(self.output_csv.get())))
do_it = messagebox.askyesno(
'', 'It may take several minutes to complete please wait')
if not do_it:
self.enable_all()
return
self.find_complete = False
filters = [
core.algorithm.SizeFilter(),
core.algorithm.CharacterFilter()
]
if (self.full_scan.get()):
filters.append(core.algorithm.FullScanner())
dup_finder = core.dup_finder.DupFinder(paths, filters)
self.status_thread = threading.Thread(target=self.update_status,
args=(dup_finder, ))
self.find_thread = threading.Thread(target=self.background_find,
args=(dup_finder, ))
self.find_thread.start()
self.status_thread.start()
def fetch_trailers( self, category=None ):
# spam log file
LOG( ">>> fetch_trailers(category: %s, rating: %s, quality: %s)" % ( repr( category ), ( "G", "PG", "PG-13", "R", "NC-17", "No Limit", )[ self.settings[ "rating" ] ], self.Fanart, ), heading=True )
ok = False
# initialize trailers list
trailers = []
# fetch source
xmlSource = self._get_xml_source()
# parse source and add our items
if ( xmlSource ):
ok = self._parse_xml_source( xmlSource, category )
# spam log file
LOG( "<<< fetch_trailers()", heading=True )
# return result
return ok
def find(self):
LOG.info("%s walk start", self.__class__.__name__)
walker = Walker()
self.__update_step(walker)
file_instances = walker.walk(self.path_list)
LOG.info("%s walk end", self.__class__.__name__)
prev_filter = self.filter_list[0]
prev_filter.set_files(file_instances)
self.total = len(file_instances)
self.__update_step(prev_filter)
prev_filter.find()
for _filter in self.filter_list[1:]:
_filter.set_files(prev_filter.filtered_files)
self.__update_step(_filter)
_filter.find()
prev_filter = _filter
def _get_xml_source( self ):
try:
xmlSource = []
# grab all xml sources
for source in ( "current.xml", "current_480p.xml", "current_720p.xml", ):
# set path and url
base_path = os.path.join( self.BASE_CURRENT_SOURCE_PATH, source )
base_url = self.BASE_CURRENT_URL % ( source, )
# get the source files date if it exists
try: date = os.path.getmtime( base_path )
except: date = 0
# we only refresh if it's been more than a day, 24hr * 60min * 60sec
refresh = ( ( time.time() - ( 24 * 60 * 60 ) ) >= date )
# only fetch source if it's been more than a day
if ( refresh ):
# open url
usock = urllib.urlopen( base_url )
else:
# open path
usock = open( base_path, "r" )
# read source
xmlSource += [ usock.read() ]
# close socket
usock.close()
# save the xmlSource for future parsing
if ( refresh ):
ok = self._save_xml_source( xmlSource[ -1 ], base_path )
# return source
return xmlSource
except Exception, e:
# oops, notify user what error occurred
LOG( str( e ), xbmc.LOGERROR )
# error so return empty string
return []
def merge_enclosed(graph, segments):
"""Merge nodes of the given graph that are on edges that do not intersects with the given segments."""
i = 0
while i < len(graph.keys()):
node = graph.keys()[i]
j = 0
altered = False
while j < len(graph[node]):
neighbour = graph[node][j]
assert (neighbour in graph)
edge = (node, neighbour)
if not any(
geometry.segment_intersection(edge, seg)
for seg in segments):
graph = merge_nodes(graph, edge[0], edge[1],
geometry.middle(*edge))
altered = True
LOG(".")
break
else:
j += 1
continue
if altered:
i = 0
else:
i += 1
return graph
def md5sum(self):
ret = setting.UNKNOWN_SYMBOL
try:
with open(self.filepath, 'rb') as fp:
chunk_size = 1024 * hashlib.md5().block_size
chksum = hashlib.md5()
while True:
chunk = fp.read(chunk_size)
if chunk:
chksum.update(chunk)
else:
break
ret = chksum.hexdigest()
except Exception as e:
LOG.error("Get {0}'s md5sum error".format(self.path))
LOG.exception(e)
return ret
def do_post():
title = request.forms.title
content = request.forms.content
id = request.forms.id
if not id:
LOG.debug('add new post...', id)
created_time = datetime.now()
modified_time = created_time
execute_sql('insert into blog values (?,?,?,?,?)' ,
(None, title, content, created_time, modified_time))
redirect('/')
else:
LOG.debug('post id is: %s', id)
modified_time = datetime.now()
execute_sql('update blog set title=?, content=?, last_modified_time=? where id=?' ,
(title, content, modified_time, id))
redirect('/post/%s' % id)
def _parse_categories( self, xmlSource, category ):
try:
# encoding
encoding = re.findall( "<\?xml version=\"[^\"]*\" encoding=\"([^\"]*)\"\?>", xmlSource[ 0 ] )[ 0 ]
# gather all trailer records <movieinfo>
trailers = re.findall( "<movieinfo id=\".+?\"><info>.+?<studio>(.*?)</studio>.+?<director>(.*?)</director>.+?(?:<cast>(.+?)</cast>)?<genre>(.+?)</genre>.+?</movieinfo>", xmlSource[ 0 + ( 2 * ( self.settings[ "trailer_quality" ] > 1 and self.settings[ "trailer_hd_only" ] ) ) ] )
# use dictionary method to filter out duplicates; set our item list
dupes = {}
# enumerate thru the trailers list and create our category list
for studio, directors, actors, genres in trailers:
# genres category
if ( category == "genres" ):
# parse genres
genres = re.findall( "<name>(.+?)</name>", genres )
# filter out duplicates
for x in genres:
dupes[ x ] = ( x, "DefaultGenre.png", None, )
elif ( category == "studios" ):
# filter out duplicates
dupes[ studio ] = ( studio, "DefaultStudios.png", None, )
elif ( category == "directors" ):
# parse directors
directors = directors.split( ", " )
# filter out duplicates
for x in directors:
dupes[ x ] = ( x, "DefaultDirector.png", None, )
elif ( category == "actors" ):
# parse actors
actors = re.findall( "<name>(.+?)</name>", actors )
# filter out duplicates
for x in actors:
dupes[ x ] = ( x, "DefaultActor.png", "special://profile/Thumbnails/Video/%s/%s" % ( xbmc.getCacheThumbName( "actor" + x )[ 0 ], xbmc.getCacheThumbName( "actor" + x ) ,), )
# grap the categories
categories = dupes.values()
# sort our list
categories.sort()
# get our media item
dirItem = DirectoryItem()
# set total items
dirItem.totalItems = len( categories )
# set as folder since these our virtual folders to filtered lists
dirItem.isFolder = True
# add settings menu item
dirItem.addContextMenuItem( "", "DUMMY TO CLEAR CONTEXT MENU" )
# enumerate thru and add our items
for title, icon, thumb in categories:
# check for cached thumb (only actors)
if ( thumb is None or not os.path.isfile( thumb ) ):
thumb = icon
# create our listitem
dirItem.listitem = xbmcgui.ListItem( title, iconImage=icon, thumbnailImage=thumb )
# set the url
dirItem.url = "%s?category=%s" % ( sys.argv[ 0 ], urllib.quote_plus( repr( "%s: %s" % ( category, unicode( title, "utf-8" ), ) ) ), )
# add item
self.MediaWindow.add( dirItem )
except Exception, e:
# oops, notify user what error occurred
LOG( str( e ), xbmc.LOGERROR )
def set_output(self, pin, output=1):
"""
:param pin:
:param output:
"""
if pin not in self.output_pins:
log.error("Pin %s was not registered as an output" % pin)
return
output_pin = self.output_pins[pin]
if output_pin['pwm_started']:
output_pin['pwm'].stop()
GPIO.output(pin, output)
log.info("Output of pin %d has been set to %d" % (pin, output))
def character(self):
chunks = list()
size = self.size
ret = setting.UNKNOWN_SYMBOL
try:
with open(self.filepath, 'rb') as f:
for i in range(NUM_CHUNKS):
f.seek(int(size / NUM_CHUNKS) * i)
chunk = f.read(NUM_BLOCKS * hashlib.md5().block_size)
chunks.append(chunk)
character_chunk = bytes()
for chunk in chunks:
character_chunk = character_chunk + chunk
ret = hashlib.md5(character_chunk).hexdigest()
except Exception as e:
LOG.error("Get {0}'s character error".format(self.path))
LOG.exception(e)
return ret
def run():
LOG.info(f"\n{L.SUCCESS} Cleaning all job entries...{L.ENDC}")
Job.objects.all().delete()
LOG.info(f"{L.SUCCESS} Running the API crawler...{L.ENDC}")
crawler()
LOG.info(f"{L.SUCCESS} Running the scraper...{L.ENDC}\n")
scraper()
earliest_job = datetime.now(tzlocal()) - timedelta(days=45)
Job.objects.filter(date__lt=earliest_job).delete()
LOG.info(f"{L.SUCCESS} Done{L.ENDC}\n")
def crawler():
counter = 1
for url_ref in config.FULL_URLS:
resp = requests.get(url_ref)
if resp.status_code == 200:
_, name = get_name(url_ref)
# Ensure folder exists
folter_path = create_folder([config.LYRICS_FOLDER, name])
# Get all links
parsed_html = BeautifulSoup(resp.content, features='html.parser')
lyrics_links = parsed_html.select('.listalbum-item a')
LOG.info(f"Number of {name.upper()} songs: {len(lyrics_links)}")
lyric_paths = [extract_link(link) for link in lyrics_links]
for lyric_path in lyric_paths:
try:
writer, song_name = get_name(lyric_path)
if name != writer:
alt_folder = create_folder(
[config.LYRICS_FOLDER, writer])
lyrics_file = alt_folder.joinpath(song_name + '.txt')
file_found = lyrics_file.is_file()
else:
writer = name
lyrics_file = folter_path.joinpath(song_name + '.txt')
file_found = lyrics_file.is_file()
if not file_found:
# url = config.BASE_URL + lyric_path
text = get_lyrics(lyric_path).strip()
LOG.info("Downloading (" + str(counter).zfill(3) +
f") [{writer}]: {song_name}")
counter += 1
with open(lyrics_file, "w") as f:
f.write(text)
time.sleep(config.CRAWLER_WAIT +
config.CRAWLER_WAIT * random.random())
except IndexError:
LOG.error(
f"Access denied while scraping: {lyric_path} \n"
f"Try increasing the waiting time.\n"
f"Finishing the scrapping for the moment. Try to access on your browser to unblock access"
)
return
except Exception as err:
print(f"ERROR: {lyric_path}: {err}")
else:
LOG.warning(f"Unable to load: {url_ref}")
def get_language_list(self):
if self.lang_code is not None:
return [self.lang_code]
expected_langs = self.extra_client_expected_langs()
LOG.debug("web client accept langs: %s", expected_langs)
lang_codes = []
for lang, priority in expected_langs:
lang_country = lang.split("-")
if len(lang_country) == 1:
lang_codes.append(lang)
continue
country = lang_country[1]
lang_codes.append("%s_%s" % (lang_country[0], country))
lang_codes.append("%s_%s" % (lang_country[0], country.swapcase()))
return lang_codes
def run_training():
df = read_data_as_df(DATA_PATH)
new_df = get_feature_df(df)
tfidf_df = get_tfidf(new_df)
X, y = preprocess_data(tfidf_df)
X_test, y_test = X.loc[X.index == 'TEST'], y.loc[y.index == 'TEST'].values
X_train, y_train = X.loc[(X.index == 'TRAIN') | (
X.index == 'VALIDATION')], y.loc[(y.index == 'TRAIN') |
(y.index == 'VALIDATION')].values
LOG.info(f"Training set: {X_train.shape}, Testing set: {X_test.shape}")
LOG.info(
f"Training set positive examples: {y_train.sum()}, Testing set positive examples: {y_test.sum()}"
)
clf_d = get_trained_models(["RF", "SGD", "LR", "SVM"], X_train, y_train)
evaluate_models(clf_d, X_train, X_test, y_train, y_test)
def restart():
LOG("Restart jobs")
db[INCIDENTS_COLLECTION_NAME].drop()
db[ADMINS_REACTION_COLLECTION_NAME].drop()
db[SERVICES_COLLECTION_NAME].drop()
db[ADMINS_COLLECTION_NAME].drop()
populate_services(read_yaml(SERVICES_YAML_PATH))
populate_admins(read_yaml(ADMINS_YAML_PATH))
#TODO stop previous jobs
hosts = [DKRON_ADDRESS]
api = Dkron(hosts)
jobs = [x['id'] for x in api.get_jobs()]
LOG("Delete jobs")
for job in jobs:
api.delete_job(job)
LOG(f'Deleted {job}')
services = get_services()
LOG("Start scheduling jobs")
for service in services:
api.apply_job({
"schedule": f'@every { service["frequency"] }s',
"name": str(service['_id']),
"timezone": "Europe/Warsaw",
"owner": "Alerting Platform",
"executor": "shell",
"executor_config": {
"command": f'python3 /app/worker.py --url {service["url"]}'
},
"processors": {
"log": {
"forward": "true"
}
},
"tags": {
"worker": "crawler:1"
}
})
LOG(f'Scheduled {service["url"]}')
def check_alive_thread():
while True:
# LOG.debug('检查客户端是否在线...')
# 不断地对keep_alive_count做递减的操作,当递减到零后,会认为客户端下线了
from protocol import CLIENT_GROUP
# 记录下线的客户端的id
offline_id_list = list()
for client in CLIENT_GROUP.get_members().values():
client.keep_alive_count -= 1
if client.keep_alive_count == 0:
offline_id_list.append(client.id)
# 移除那些离线的客户端
for c_id in offline_id_list:
if CLIENT_GROUP.remove(c_id):
LOG.debug('id: {}的客户端已经离线'.format(c_id))
sleep(CHECK_INTERVAL)
def main():
path = sys.argv[1]
LOG.info("Start to find duplicated files on {0}".format(path))
if os.path.isfile(path):
start_time = time.time()
print (File(path).md5sum)
print (File(path).size)
end_time = time.time()
print (end_time - start_time)
else:
start_time = time.time()
filters = [
core.algorithm.SizeFilter(),
core.algorithm.CharacterFilter()
]
dup_finder = core.dup_finder.DupFinder([path], filters)
dup_finder.find()
end_time = time.time()
#dup_finder.dump2file("output.txt")
dup_finder.dump2csv("output.csv")
print (end_time - start_time)
print (utils.size_renderer(dup_finder.dup_size))
def main():
global PROGRAM_CONFIG
# 加载配置
config = load_config()
if config:
PROGRAM_CONFIG = config
# 获取日志等级
if PROGRAM_CONFIG['log_level'] == 'DEBUG':
config_logging(logging.DEBUG, PROGRAM_CONFIG['log_name'])
elif PROGRAM_CONFIG['log_level'] == 'INFO':
config_logging(logging.debug, PROGRAM_CONFIG['log_name'])
elif PROGRAM_CONFIG['log_level'] == 'ERROR':
config_logging(logging.ERROR, PROGRAM_CONFIG['log_name'])
elif PROGRAM_CONFIG['log_level'] == 'WARN':
config_logging(logging.WARN, PROGRAM_CONFIG['log_name'])
else:
config_logging()
server = ProxyServer(PROGRAM_CONFIG['listen_port'])
# 用于检查客户端在线状态的线程
check_thread = Thread(target=check_alive_thread)
try:
print('启动检查客户端在线状态的线程')
check_thread.start()
server.start()
except Exception as err:
LOG.error(str(err))
finally:
print('写入配置信息')
save_config(PROGRAM_CONFIG)
print('退出程序')
def initial_message(msg):
log.info('Initial Message from connected Client: %s' % msg['data'])
def on_connect():
log.info('Client connected!')
def on_disconnect():
log.info('Client disconnected!')
def signal_handler(signal, frame):
print ""
log.info("Shutting down ...")
# do some cleaning stuff
gpio_controller.cleanup()
sys.exit(0);
def set_files(self, files):
LOG.debug("%s set_files", self.__class__.__name__)
self.files = files
def _decode_single_body(self):
self.body = self.body.strip()
cte = self.headers.get('Content-Transfer-Encoding', '').lower()
if 'quoted-printable' in cte:
LOG.debug("Detected quoted-printable encoding, decoding")
self.body = quopri.decodestring(self.body)
if 'base64' in cte:
LOG.debug("Detected base64 encoding, decoding")
try:
self.body = base64.decodestring(self.body)
except base64.binascii.Error:
LOG.info("base64 decoder failed, trying partial decoding")
self.body = base64_partial_decode(self.body)
LOG.debug("Detected charset: %s", self.charset)
try:
self.body = self.body.decode(
validate_charset(self.charset) and self.charset or 'ascii',
'strict'
)
except UnicodeDecodeError:
LOG.info('Error during strict decoding')
self.email_stats['charset_errors'] = 1
self.body = self.body.decode(
validate_charset(self.charset) and self.charset or 'ascii',
'ignore'
)
if self._guess_html():
LOG.debug("Message recognized as HTML")
self._parse_html()
else:
LOG.debug("Message recognized as plaintext")
def delete(id):
LOG.info('delete blog #%s', id)
execute_sql('delete from blog where id =?', (id,))
redirect('/')
