def __init__(self):
propFile = None
print 'SM_PROP_DIR = ' + str(os.environ.get('SM_PROP_DIR'))
if os.environ.has_key('SM_PROP_DIR'):
try:
propFile = os.environ.get('SM_PROP_DIR') + '\propertyfile.dat'
except:
print 'failed to form propFile, aborting.'
sys.exit()
else:
try:
print 'attempting to open propertyfile.dat =' + propFile
myfile = open(propFile, 'r')
except:
logger.err("failed to open propertyfile = " + propFile + '. aborting')
sys.exit()
else:
self.properties = {}
property = {}
for line in myfile:
l = line[0:len(line)-1]
elements = l.split('=')
property = {str(elements[0]):str(elements[1])}
self.properties.update(property)
else:
print 'SM_PROP_DIR envvar not defined, aborting'
sys.exit()
def set_config_file(self, file_path):
if not isinstance(file_path, str):
logger.err(str(file_path) + " is not a valid parameter!")
return PARAMETER_INVALID
logger.info("Set config file to " + self.__config_file)
self.__config_file = file_path
def lineReceived( self, line ):
logger.debug( 'Line In: %s' % line )
if not line.strip(): # prepare to and give control to action selector
self.transport.loseConnection()
callee = 'null'
# only 'on_hangup' must send MEMBERINTERFACE as arg_1
if self.environment.has_key( 'agi_arg_1' ):
if self.environment[ 'agi_arg_1' ] != '':
callee = self.environment[ 'agi_arg_1' ]
# extracting callee extension with regexp
callee = re.sub( '[a-zA-Z]+/', '', callee )
else:
# in case of outgoing call we're just get dnid as callee
# also this will be set in case of not answered incoming call
callee = self.environment[ 'agi_dnid' ]
self.factory.actionSelect( self.environment[ 'agi_network_script' ],
self.environment[ 'agi_uniqueid' ],
self.environment[ 'agi_callerid' ],
callee )
else: # fetching environmental variables
try:
key, value = line.split( ':', 1 )
value = value[1:].rstrip( '\n' ).rstrip( '\r' )
except ValueError, err:
logger.err( "Invalid variable line: %s", line )
else:
def lineReceived(self, line):
logger.debug('Line In: %s' % line)
if not line.strip(): # prepare to and give control to action selector
self.transport.loseConnection()
callee = 'null'
# only 'on_hangup' must send MEMBERINTERFACE as arg_1
if self.environment.has_key('agi_arg_1'):
if self.environment['agi_arg_1'] != '':
callee = self.environment['agi_arg_1']
# extracting callee extension with regexp
callee = re.sub('[a-zA-Z]+/', '', callee)
else:
# in case of outgoing call we're just get dnid as callee
# also this will be set in case of not answered incoming call
callee = self.environment['agi_dnid']
self.factory.actionSelect(self.environment['agi_network_script'],
self.environment['agi_uniqueid'],
self.environment['agi_callerid'], callee)
else: # fetching environmental variables
try:
key, value = line.split(':', 1)
value = value[1:].rstrip('\n').rstrip('\r')
except ValueError, err:
logger.err("Invalid variable line: %s", line)
else:
def parse(self, line, db):
node = self.tagger.parseToNode(line.encode('utf-8'))
# accumulate words until end of stream or sentence
while node:
if node.stat == MeCab.MECAB_BOS_NODE:
sentence = u''
data = []
elif node.stat == MeCab.MECAB_NOR_NODE or node.stat == MeCab.MECAB_UNK_NODE:
try:
word = node.surface.decode('utf-8')
fields = node.feature.decode('utf-8').split(',')
# get part-of-speech features
pos = fields[0:self.fields]
if fields[6] != u'*': # if root form is available, use it
root = fields[6]
else:
root = word
sentence = sentence + word
data.append([root] + pos)
if pos[0] == u'記号' and pos[1] == u'句点' \
or pos[0] == u'記号' and word == u'！' \
or pos[0] == u'記号' and word == u'？':
self.insert(data, sentence, db)
sentence = u''
data = []
except UnicodeDecodeError as e:
logger.err('could not decode %s' % node.surface)
elif node.stat == MeCab.MECAB_EOS_NODE:
self.insert(data, sentence, db)
node = node.next
def ingestRecipiesIngredients(self):
logger.started()
tmp = []
if self.pm.get('USE_DB') == 'N':
logger.info('reading from file')
try:
myfile = open(self.pm.get('RECIPE_INGREDIENTS_FILE'), 'r')
except:
logger.err("failed to find recipesingredients, aborting")
sys.exit()
else:
self.recipesIngredients = {}
for line in myfile:
l = line[0:len(line)-1]
elements = l.split(',')
recipeId = elements[0]
ingredId = elements[1]
qty = elements[2]
if recipeId in self.recipesIngredients:
logger.info('recipeId=' + str(recipeId) + ' exists')
else:
logger.info('adding recipeId=' + str(recipeId))
self.recipesIngredients[recipeId] = {}
self.recipesIngredients[recipeId][ingredId] = int(qty)
else:
# initialize a cursor on the connection. a cursor is essentially a reference to a SELECT stmt and the rows that it returns once executed
logger.info('ingestRecipeIngreds: reading from database')
cur = self.conn.cursor()
# load up the cursor with the SELECT stmt and execute it
cur.execute('select recipe_id, ingred_id, qty from recipeingredients')
# iterate thru the rows of the cursor via the cur.fetchall method. that method returns a list for each row. each such list contains values corresponding to the columns in the SELECT clause
for (row) in cur.fetchall():
tmp.append({'rid':row[0],'iid':row[1],'q':row[2]})
self.recipesIngredients = {}
for ri in tmp:
if ri['rid'] in self.recipesIngredients:
logger.info('recipeId =' + str(ri['rid']) + ' exists')
self.recipesIngredients[ri['rid']][ri['iid']] = int(ri['q'])
else:
logger.info('adding recipeId =' + str(ri['rid']) + ' ingred ID =' + str(ri['iid']))
self.recipesIngredients[ri['rid']] = {}
self.recipesIngredients[ri['rid']][ri['iid']] = int(ri['q'])
#self.dumpRecipesIngredients()
logger.completed()
def change_log_level(self, new_level):
if not isinstance(new_level, str):
logger.err(str(new_level) + " is not a valid parameter!")
return PARAMETER_INVALID
self.__log_level = new_level
logger.set_level(new_level)
logger.force_show_info("Change log level to '" + new_level + "'")
return new_level
def change_log_format(self, new_format):
if not isinstance(new_format, str):
logger.err(str(new_format) + " is not a valid parameter!")
return PARAMETER_INVALID
self.__log_format = new_format
logger.set_format(new_format)
logger.force_show_info("Change log format to '" + new_format + "'")
return new_format
def enable_silence_mode(self, mode):
if isinstance(mode, str):
mode = bool(mode)
if not isinstance(mode, bool):
logger.err(str(mode) + " is not a valid parameter!")
return PARAMETER_INVALID
self.__is_silence = mode
enable_or_disable = "Enable" if mode else "Disable"
logger.info(enable_or_disable + " silence mode")
return mode
def replace_gaiji(self, gaiji_match):
jis_plane = int(gaiji_match.group('JisPlane'))
jis_row = int(gaiji_match.group('JisRow'))
jis_col = int(gaiji_match.group('JisCol'))
gaiji_code = 0x100 * (jis_row + 0x20) + (jis_col + 0x20)
if jis_plane == 2:
gaiji_code = gaiji_code + 0x8080
try:
#logger.err('replaced %s with %s' % (gaiji_match.group(0), self.gaiji_codes[gaiji_code]))
return self.gaiji_codes[gaiji_code]
except KeyError:
logger.err('found gaiji with no equivalent: %s' % gaiji_match.group(0))
return u''
def analyze(filename, formatter, parser, encoding, db):
logger.out('reading %s' % filename)
formatter.new_file()
try:
fp = codecs.open(filename, 'r', encoding)
except IOError as e:
logger.err('error opening %s: %s' % (filename, e))
else:
with fp:
# process all files line by line
for line in fp:
trimmed_line = formatter.trim(line)
mecab_data = parser.parse(trimmed_line, db)
def __init__(self, path):
self.path = path
try:
myfile = open(path)
except:
logger.err('invalid path: ' + path)
else:
lines = []
txt = ''
for line in myfile:
lines.append(line)
for l in lines:
txt = txt + l
payload.payload.__init__(self, txt)
def initRecipes(self):
logger.started()
tmp = []
if self.pm.get('USE_DB') == 'Y':
# initialize a cursor on the connection. a cursor is essentially a reference to a SELECT stmt and the rows that it returns once executed
logger.info('using database')
cur = self.conn.cursor()
# load up the cursor with the SELECT stmt and execute it
cur.execute('select recipe_id, recipe_name from recipes')
for (row) in cur.fetchall():
tmp.append({'id':row[0], 'name':row[1]})
else:
logger.info('using file')
try:
myfile = open(self.pm.get('RECIPES_FILE'), 'r')
except:
logger.err("failed to find recipes, aborting")
sys.exit()
else:
for line in myfile:
l = line[0:len(line)-1]
elements = l.split(',')
tmp.append({'id':elements[0], 'name':elements[1]})
for r in tmp:
temp_ingreds = []
try:
temp_ingreds = self.recipesIngredients[r['id']]
logger.info('added ingred, recipe id=' + str(r['id']))
except:
logger.info('no ingred found, recipe id=' + str(r['id']))
temp_ingreds = {}
recipe = {
'id':int(r['id']),
'name':r['name'],
'ingreds':temp_ingreds,
'is_make':IntVar()
}
self.recipes.append(recipe)
logger.completed()
def change_output_folder(self, new_path):
if not isinstance(new_path, str):
logger.err(str(new_path) + " is not a valid parameter!")
return PARAMETER_INVALID
output_folder = format_path(new_path)
self.__output_folder = output_folder
logger.info("Change output folder to " + output_folder)
# If the output folder has already existed, warn user to confirm
if os.path.exists(output_folder) and os.listdir(output_folder):
confirm = call_user_confirm(
output_folder +
" has already existed, still continue to extract files here?")
if not confirm:
return exit(USER_BREAK_EXIT)
return output_folder
def change_remote_folder(self, new_path):
if not isinstance(new_path, str):
logger.err(str(new_path) + " is not a valid parameter!")
return PARAMETER_INVALID
formatted_new_path = format_path(new_path)
# If failed, it'll raise an error
result = DriveMapper.map(new_path)
if result == SUCCESS:
self.__remote_folder = formatted_new_path
logger.info("Change remote folder to " + self.__remote_folder +
" successfully.")
return concat_path(DriveMapper.get_current_drive_letter(),
FILE_PATH_SEP)
return formatted_new_path
def try_to_map(cls, path, drive_letter=None):
if drive_letter is None:
drive_letter = cls.__cur_drive_letter
p = Popen('net use ' + drive_letter + ' "' + path + '"',
stdout=PIPE,
stderr=STDOUT,
shell=True)
p.wait()
if p.returncode != SUCCESS:
logger.err("Fail to map the drive " + drive_letter)
return MAP_DRIVE_FAILURE
else:
logger.info("Succeed to map the drive " + drive_letter)
cls.__has_been_mapped = True
cls.__cur_drive_letter = drive_letter
cls.__cur_remote_path = path
return SUCCESS
def initIngreds(self):
logger.started()
tmp = []
if self.pm.get('USE_DB') == 'N':
try:
path = self.pm.get('INGREDIENTS_FILE')
self.l.info('path=' + path)
myfile = open(path, 'r')
except:
logger.err( "failed to find ingredients file, aborting")
sys.exit()
else:
for line in myfile:
l = line[0:len(line)-1]
elements = l.split(',')
tmp.append({'id':elements[0], 'name':elements[1], 'units':elements[2]})
else:
# initialize a cursor on the connection. a cursor is essentially a reference to a SELECT stmt and the rows that it returns once executed
self.conn.connect()
logger.info('connection to db')
cur = self.conn.cursor()
# load up the cursor with the SELECT stmt and execute it
cur.execute("SELECT ingred_id, ingred_name, ingred_uom from ingredients")
# iterate thru the rows of the cursor via the cur.fetchall method. that method returns a list for each row. each such list contains values corresponding to the columns in the SELECT clause
for (row) in cur.fetchall():
tmp.append({'id':row[0], 'name':row[1], 'units':row[2]})
for i in tmp:
ingred = {'name':i['name'],
'units':i['units'],
'is_needed':IntVar(),
'is_buy':IntVar(),
'total_req_qty':0,
'total_req_qty_disp':StringVar()}
self.ingreds[i['id']] = ingred
logger.completed()
def __init__(self):
self.linecount = 0
# load gaiji codes
self.gaiji_codes = {}
gaijifile = os.path.join(config.get_basedir(), config.gaijifile)
try:
fp = open(gaijifile, 'r')
except IOError as e:
logger.err('error opening gaiji codes file: %s' % (filename, e))
else:
with fp:
for line in fp:
match = re.match(ur'^0x(?P<JisCode>[0-9A-Fa-f]+)\tU\+(?P<UtfCode>[0-9a-fA-F]+)\+?(?P<UtfCode2>[0-9a-fA-F]+)?', line)
if match:
gaiji_code = int(match.group('JisCode'), 16)
utf_char = unichr(int(match.group('UtfCode'), 16))
if match.group('UtfCode2'): # 2-character representation
utf_char = utf_char + unichr(int(match.group('UtfCode2'), 16))
self.gaiji_codes[gaiji_code] = utf_char
def try_to_cancel_mapping(cls, drive_letter=None):
if drive_letter is None:
drive_letter = cls.__cur_drive_letter
p = Popen('net use ' + drive_letter + ' /del /y',
stdout=PIPE,
stderr=STDOUT,
shell=True)
p.wait()
if p.returncode != SUCCESS:
logger.err("Fail to cancel the mapping of drive " + drive_letter)
return CANCEL_MAPPING_DRIVE_FAILURE
else:
logger.info("Succeed to cancel the mapping of drive " +
drive_letter)
cls.__has_been_mapped = False
cls.__cur_drive_letter = None
cls.__cur_remote_path = None
return SUCCESS
def _check_the_path_okay_to_move(path, filename):
file_full_path = concat_path(path, filename)
if not is_parent_all_folder_or_not_exist(file_full_path):
logger.err(
"Remove the existing file failed. " +
"There's a file with the name same with one folder name in the parent path, "
+ "please check it manually!")
return False
else:
if os.path.exists(file_full_path):
logger.warn("A file with the same name '" + filename +
"' already exists in " + path)
try:
remove_file_or_folder(file_full_path)
except OSError as e:
logger.exception(e)
logger.err("Remove the existing file failed.")
return False
else:
logger.info("Remove the existing file successfully.")
return True
def _queryFail(self, results):
# TODO: need something more useful here
logger.err("Query failed. Something bad happened to database?")
def main():
# parse command line options
try:
opts, args = getopt.getopt(sys.argv[1:], 'hn:t:', ['help','number=','tablename='])
except getopt.error as opterr:
logger.err(opterr)
logger.err('for help use --help')
sys.exit(2)
# process config and options
list_number = config.list_number
tablename = config.tablename
for o, a in opts:
if o in ('-h', '--help'):
logger.out(__doc__)
sys.exit(0)
if o in ('-n', '--number'):
try:
top_number = int(a)
except ValueError:
logger.err('invalid argument for top number: %s' % a)
sys.exit(2)
if list_number <= 0:
logger.err('invalid top number: %s' % list_number)
sys.exit(2)
if o in ('-t', '--tablename'):
tablename = a
if not re.match(r'^[_a-zA-Z][_a-zA-Z0-9]*$', tablename):
logger.err('invalid table name: %s' % tablename)
sys.exit(2)
# open gui with database
try:
db = database.Database(tablename)
with db:
ui = gui.FreqGUI(db, list_number)
ui.show()
except sqlite3.Error as e:
logger.err('database error: %s' % e)
def onUnrecognised(self, id, caller, callee):
logger.err(
"Unrecognised AGI script name. Check your Asterisk dialplan configuration."
)
def main():
basedir = config.get_basedir()
# parse command line options
try:
opts, args = getopt.getopt(sys.argv[1:], 'hf:e:o:rdt:s', ['help','format=','encoding=', 'droptable', 'recursive', 'tablename=', 'sentences'])
except getopt.error as opterr:
logger.err(opterr)
logger.err('for help use --help')
sys.exit(2)
# process config and options
formatter = config.formatter
encoding = config.encoding
tablename = config.tablename
drop = False
recursive = False
sentences = False
for o, a in opts:
if o in ('-h', '--help'):
logger.out(__doc__)
sys.exit(0)
if o in ('-f', '--format'):
formatter = a
if formatter not in ('plain', 'aozora', 'html'):
logger.err('format not supported: %s' % formatter)
sys.exit(2)
if o in ('-e', '--encoding'):
encoding = a
try:
codecs.lookup(encoding)
except LookupError:
logger.err('encoding not found: %s' % encoding)
sys.exit(2)
if o in ('-d', '--droptable'):
drop = True
if o in ('-s', '--sentences'):
sentences = True
if o in ('-t', '--tablename'):
tablename = a
if not re.match(r'^[_a-zA-Z][_a-zA-Z0-9]*$', tablename):
logger.err('invalid table name: %s' % tablename)
sys.exit(2)
if o in ('-r', '--recursive'):
recursive = True
# create formatter and parser
if(formatter == 'aozora'):
formatter = formats.AozoraFormat()
elif(formatter == 'html'):
formatter = formats.HTMLFormat()
else:
formatter = formats.Format()
parser = mecab.PyMeCab(sentences)
# access database
try:
db = database.Database(tablename)
with db:
if(drop):
db.drop_table()
db.create_table()
# process files
logger.out('analyzing text files')
if recursive:
for dirname in args:
for dirpath, dirs, files in os.walk(dirname):
logger.out('going through directory %s' % dirpath)
for filename in files:
analyze(os.path.join(dirpath, filename), formatter, parser, encoding, db)
else:
for filename in args:
analyze(filename, formatter, parser, encoding, db)
logger.out('done analyzing')
except sqlite3.Error as e:
logger.err('database error: %s' % e)
def onUnrecognised( self, id, caller, callee ):
logger.err( "Unrecognised AGI script name. Check your Asterisk dialplan configuration." )
def _queryFail( self, results ):
# TODO: need something more useful here
logger.err( "Query failed. Something bad happened to database?" )
def test():
logger.info('info: No active formats remain. Continue?')
logger.suc('success: No active formats remain. Continue?')
logger.warn('warning: No active formats remain. Continue?')
logger.err('error: No active formats remain. Continue?')
logger.fail('fail: No active formats remain. Continue?')
def main():
welcome()
parse_args()
if not os.path.exists(gl.get_config_file()):
logger.err("Config file " + gl.get_config_file() + " does not exist!")
return NOT_FOUND_CONFIG_FILE
with open(gl.get_config_file(), "r",
encoding='UTF-8') as config_file_handler:
config_lines = config_file_handler.readlines()
# Delete the UTF-8 BOM characters
if config_lines[0][:3] == codecs.BOM_UTF8:
config_lines[0] = config_lines[0][3:]
cur_src_folder = gl.get_remote_folder()
cur_extracting_folder = gl.get_output_folder()
cur_line_num = 0
total_file_count = 0
for cur_line in config_lines:
cur_line_num += 1
cur_line = cur_line.strip()
# Empty line / skipping line
if cur_line == "" or cur_line.startswith(SKIP_THIS_LINE_FLAG):
continue
# Parameter line (The line like '--log-format = normal')
elif is_full_matched(PARAMETER_LINE_PATTERN, cur_line):
split_arr = cur_line.split(PARAMETER_LINE_FLAG)
arg_key_part = split_arr[0][2:].strip()
arg_value_part = split_arr[1].strip()
if arg_key_part == PARA_KEY_SILENCE_MODE:
gl.enable_silence_mode(arg_value_part)
elif arg_key_part == PARA_KEY_REMOTE_FOLDER:
cur_src_folder = gl.change_remote_folder(arg_value_part)
elif arg_key_part == PARA_KEY_OUTPUT_FOLDER:
cur_extracting_folder = gl.change_output_folder(
arg_value_part)
elif arg_key_part == PARA_KEY_LOG_LEVEL:
gl.change_log_level(arg_value_part)
elif arg_key_part == PARA_KEY_LOG_FORMAT:
gl.change_log_format(arg_value_part)
else:
logger.err("No parameter named " + arg_key_part,
cur_line_num)
continue
# Changing path line (start with '>')
elif cur_line.startswith(CHANGE_PATH_LINE_FLAG):
cur_line = cur_line.replace(CHANGE_PATH_LINE_FLAG, "").strip()
cur_extracting_folder = format_path(
concat_path(gl.get_output_folder(), cur_line))
logger.info("Change current extracting folder to " +
cur_extracting_folder)
continue
# File line
else:
cur_remote_path = gl.get_remote_folder()
if is_remote_path(
cur_remote_path
) and not DriveMapper.has_been_mapped(cur_remote_path):
DriveMapper.map(cur_remote_path, force=True)
try:
total_file_count += Extractor.extract(
cur_line, cur_src_folder, cur_extracting_folder)
except CustomError as custom_error:
logger.err(custom_error.message, cur_line_num)
logger.info("Finish to extracted all of the " +
format_plural(total_file_count, "file") + ".")
# Try to cancel the drive mapping
if DriveMapper.has_been_mapped():
if DriveMapper.try_to_cancel_mapping() != SUCCESS:
logger.warn("Maybe you need to cancel it manually.")
# If error occurred, show the warning info
if logger.has_error_occurred:
logger.warn(
"All finished, but several errors occurred during the process, " +
"please check " + logger.filename + " to confirm.")
else:
logger.info("All finished without any error!")
return SUCCESS
