def test_path_exists():
s = '\n'
s += 'a.txt = ' + str(util.path_exists('a.txt')) + '\n'
s += 'b.txt = ' + str(util.path_exists('b.txt')) + '\n'
s += 'dir0 = ' + str(util.path_exists('dir0')) + '\n'
s += 'dir1 = ' + str(util.path_exists('dir1')) + '\n'
return s
def write_output(output):
for about_file_location, context in output:
about_file_location = add_unc(about_file_location)
if path_exists(about_file_location):
os.remove(about_file_location)
with open(about_file_location, 'wb') as output_file:
output_file.write(context)
def __init__(self, init_params, friendly_params, model_dir, warm_start_dir,
mode):
self.init_params = init_params
self.friendly_params = friendly_params
self.model_dir = model_dir
self.warm_start_dir = warm_start_dir
self.graph = tf.Graph()
with self.graph.as_default():
input_fn = self.init_params[mode + "_input_fn"](
self.friendly_params)
inpt = input_fn()
self.model = self.init_params["model_fn"](inpt[0], inpt[1], mode,
self.friendly_params)
self.model_mode = mode
hooks = []
if mode == "train":
self.saver = tf.train.Saver()
self.checkpoint_saver = tf.train.CheckpointSaverHook(
checkpoint_dir=self.model_dir,
save_steps=self.friendly_params["micro_step"],
saver=self.saver)
hooks.append(self.checkpoint_saver)
if self.init_params.get('profile', False):
profiler = tf.train.ProfilerHook(save_steps=100,
output_dir=self.model_dir)
hooks.append(profiler)
if tf.summary.merge_all() is not None:
hooks.append(
tf.train.SummarySaverHook(
save_secs=10,
output_dir=self.model_dir,
summary_op=tf.summary.merge_all()))
# Transparent across GCS and local paths
if path_exists(self.model_dir):
# We should resume from that location
load_dir = self.model_dir
else:
# We should try to warm start
load_dir = self.warm_start_dir
logger.debug(
"model_dir: {} warm_start_dir: {} load from:{}".format(
self.model_dir, self.warm_start_dir, load_dir))
self.sess = tf.train.SingularMonitoredSession(
hooks=hooks, checkpoint_dir=load_dir)
def copy_files(gen_location, files_list):
"""
Copy the files into the gen_location
"""
# FIXME : why use a static and not a regular function?
for file_path, component_path in files_list:
output_file_path = join(gen_location, component_path)
if not path_exists(output_file_path):
makedirs(output_file_path)
shutil.copy2(file_path, output_file_path)
def write_licenses(self, license_context_list):
for gen_license_path, license_context in license_context_list:
try:
if not path_exists(dirname(gen_license_path)):
makedirs(dirname(gen_license_path))
with open(gen_license_path, 'wb') as output:
output.write(license_context)
except Exception:
err = Error(UNKNOWN, 'Unknown', gen_license_path,
'Something is wrong.')
self.errors.append(err)
def report_feedback(userid):
"""
"""
print "[statistics] Starting feedback send"
fbpath = makepath('feedback.json')
if not path_exists(fbpath):
feedback = []
else:
feedback = jsonload(fbpath)
if len(feedback) > 0:
data = {'feedback' : feedback, 'userid' : userid}
network_POST('/stats/report_feedback', data)
jsondump(makepath('feedback.json'), [])
print '[statistics] Finished feedback send'
altpath = makepath('feedback.txt')
if path_exists(altpath):
feedback = fileload(altpath)
data = {'feedback' : feedback, 'userid' : userid}
network_POST('/stats/report_legacy_feedback', data)
command('rm', altpath)
def verify_files_existence(self, input_list, project_dir, file_in_project):
"""
Verify the existence of a file pointed to by ' _file' suffixed fields
such as 'license_text_file'.
"""
files_list = []
# Get all the dictionary keys
column_keys = input_list[0].keys()
# Get all the keys that ends with _file except for the 'about_file'
file_keys = []
for key in column_keys:
if key.endswith('_file') and key != 'about_file':
file_keys.append(key)
# FIXME: this loop is too complex
for line in input_list:
for file_key in file_keys:
if not line[file_key]:
continue
file_path_list = []
file_value = []
file_location = line['about_file']
if file_location.startswith('/'):
file_location = file_location.partition('/')[2]
about_parent_dir = dirname(file_location)
if file_in_project:
if '\n' in line[file_key]:
file_value = line[file_key].split('\n')
else:
file_value.append(line[file_key])
for value in file_value:
if file_location.endswith('/'):
about_parent_dir = normpath(dirname(join(file_location, value)))
file_path_list.append(join(project_dir, dirname(file_location), value))
else:
if '\n' in line[file_key]:
file_value = line[file_key].split('\n')
else:
file_value.append(line[file_key])
for value in file_value:
file_path_list.append(join(project_dir, value))
for path in file_path_list:
if path_exists(path):
files_list.append((path, about_parent_dir))
else:
self.warnings.append(Warn(FILE, file_key, path, 'File does not exist.'))
return files_list
def _get_run_directory_from_possible_paths(possible_paths):
run_in_path = None
# Get the last file argument and also get the base path for run_in_path
if len(possible_paths):
for index, path in possible_paths:
# Use the first existing path as the run_in_path variable
# This ensures using the path to the executable, not the argument
if util.path_exists(path):
run_in_path = os.path.dirname(util.wintounix(path))
break
last_file_arg = util.unixtowin(possible_paths[-1][1])
last_file_arg_index = possible_paths[-1][0]
else:
last_file_arg = None
last_file_arg_index = None
return (last_file_arg, last_file_arg_index, run_in_path)
def process_dje_licenses(self, dje_license_list, dje_license_dict, output_path):
license_list_context = []
for gen_path, license_name in dje_license_list:
lic = license_name
if gen_path.startswith('/'):
gen_path = gen_path.partition('/')[2]
if lic:
license_key = dje_license_dict[lic][0]
gen_license_path = join(output_path, gen_path, license_key) + '.LICENSE'
if not path_exists(gen_license_path) and not self.extract_license_api_error:
context = dje_license_dict[lic][1]
if context:
gen_path_context = []
gen_path_context.append(gen_license_path)
gen_path_context.append(context.encode('utf8'))
license_list_context.append(gen_path_context)
return license_list_context
def get_dje_license_list(self, gen_location, input_list, gen_license, dje_license_dict):
# FIXME : this is too complex
license_output_list = []
for line in input_list:
try:
# If there is value in 'license_text_file', the tool will not
# update/overwrite the 'license_text_file' with
# 'dje_license_key'
if line['license_text_file']:
file_location = line['about_file']
about_parent_dir = dirname(file_location)
license_text_file = line['license_text_file']
license_file = normpath(gen_location.rpartition('/')[0] + join(about_parent_dir, license_text_file))
if not path_exists(license_file):
self.errors.append(Error(FILE, 'license_text_file', license_file, "The 'license_text_file' does not exist."))
else:
if gen_license and not self.extract_license_api_error:
if line['dje_license_key']:
license_output_list.append(self.gen_license_list(line))
lic_name = line['dje_license_name']
line['license_text_file'] = dje_license_dict[lic_name][0] + '.LICENSE'
else:
self.warnings.append(Warn(VALUE, 'dje_license_key', '',
"Missing 'dje_license_key' for " + line['about_file']))
# This except condition will force the tool to create the
# 'license_text_file' key column from the self.gen_license_list(line)
except Exception:
# FIXME: this is too complex
if gen_license:
if line['dje_license_key']:
license_output_list.append(self.gen_license_list(line))
lic_name = line['dje_license_name']
if lic_name:
line['license_text_file'] = dje_license_dict[lic_name][0] + '.LICENSE'
else:
self.warnings.append(Warn(VALUE, 'dje_license_key', '',
"Missing 'dje_license_key' for " + line['about_file']))
return license_output_list
def insert(connection, cursor):
'''receives: connection, cursor from psycopg2,
reads from given file and inserts line by line
by certain rules
returns None'''
util.validate_system_argvs(2, argv=argv)
if not util.path_exists(util.get_cwd() + '/' + argv[2]):
util.error('file_doesnt_exist', argv[2])
else:
with open(argv[2]) as log_file:
while True:
log_file_entry = log_file.readline()
if log_file_entry:
data = util.split_line(log_file_entry, log_file)
insert_line(data, cursor)
else:
connection.commit()
break
return
def parse_command(command, env=None, run_in_path=None):
"""
Parse a command, outputting a dict containing executable, arguments,
environment and so on.
Commands that have better XDG replacements are replaced.
command can be list (preferably) or string,"""
run_in_path_override = run_in_path
# If command is given as a string
if type(command) in (str, unicode):
if util.path_exists(command):
command = [command]
elif util.path_exists(util.unescape_string(command)):
command = [util.unescape_string(command)]
else:
command = util.string_split(
command,
retain_defines=True,
remove_escapes=True
)
else:
# This is a list, meaning it has to be preparsed, don't change it
command = command[:] # make a copy, not a link
# These variables are reported back in a dict
wine_prefix = None # WINEPREFIX, will be set in environment
executable = None # The executable to run, usually returns wine
internal = False # If True, don't monitor since nothing should go wrong
if env is None or type(env) is not dict:
env = common.ENV.copy() # Self-explanatory - except that any 'env'-command will alter it
run_in_path = None # Start the command in this path
name = None # Name will be set to something explanatory, like "Console"
# Remove double backslashes
command = [ i.replace('\\\\', '\\') for i in command ]
print("Command split to", command)
# Move any env definitions to the env dict
if command[0] == 'env':
for index, arg in enumerate(command[1:]):
if '=' in arg:
key = arg.split('=')[0]
value = '='.join(arg.split('=')[1:])
if len(value) and value[0] in ('"', "'"):
value = value[1:-1]
env[key] = value
else:
break
command = command[1+index:]
if command[0] == 'wine' or command[0] == common.ENV['WINE'] or re.search(r'/.*/wine$', command[0]) is not None:
del command[0]
executable = common.ENV['WINE']
if len(command) == 1:
command[0] = util.string_remove_escapes(command[0])
# Find all file and directory paths in the command
possible_paths = []
for index, i in enumerate(command):
if i[1:3] == ':\\':
possible_paths.append((index, i))
elif i[0] == '/':
if os.path.exists(i):
possible_paths.append((index, i))
elif os.path.exists(util.unescape_string(i)):
possible_paths.append((index, util.unescape_string(i)))
command[index] = util.unescape_string(i)
#print("POSSIBLE PATHS:",possible_paths)
last_file_arg, last_file_arg_index, run_in_path = (
_get_run_directory_from_possible_paths(possible_paths)
)
print("possible paths: ",possible_paths)
first_command = command[0].lower()
#print("FIRST:",first_command)
#print("LAST EXE:", last_file_arg)
# Known, simple commands that don't require parsing
if first_command in ('sh', 'env'):
# Don't parse this command, it's shell magic
executable = command[1]
command = command[2:]
name = "Shell command"
elif first_command == 'winetricks':
# Get the path for winetricks, if it exists
winetricks = winetricks_installed()
if winetricks:
command[0] = winetricks
internal = True
# More advanced/troublesome commands
else:
command_lowercase = [ i.lower() for i in command ]
# If internal command
if (
first_command.split('.')[-1] in common.BUILTIN_EXECUTABLES
) or (
first_command[2:].startswith('\\windows\\command\\') and
first_command.split('\\')[-1].split('.')[0] in common.BUILTIN_EXECUTABLES
):
first_command_name = first_command.split('\\')[-1].split('.')[0]
# Check for command being for a program that should be run in a command console
if first_command_name == 'cmd':
if len(command) > 1:
name = "Console: {0}".format(command[1])
else:
name = "Console"
command = [
'wineconsole', '--backend=user', 'cmd'
] + command[1:]
elif first_command_name == 'regedit':
name = "Regedit"
internal = True
# Override start.exe opening directories or URL, use xdg-open
elif first_command_name == 'start':
#print(command_lowercase)
if '/unix' in command_lowercase:
# This is already a unix path, don't convert
# Select the argument after '/unix'
path = command[
command_lowercase.index('/unix')+1
]
# Convert that argument to a UNIX path
path = util.wintounix(util.unixtowin(util.string_remove_escapes(path)))
else:
if len(possible_paths):
# This is supposedly a Windows path, use the grabbed path
path = util.wintounix(possible_paths[0][1])
elif len(command) > 1:
# We couldn't grab a path, use first argument
path = command[1]
else:
# All is lost, assume 'start' is the actual command
path = command[0]
try:
#link = parsers.read_lnk_file(path)
link = binary.windows_link(path)
if link and 'location' in link:
path = link['location']
if 'work dir' in link:
run_in_path_override = link['work dir']
except IOError:
pass
# Only use this path if it's actually a URL or directory
if (
'://' in path[:8] or
os.path.isdir(util.wintounix(path))
):
print("start.exe was asked to open a directory or URL, use xdg-open instead")
executable = 'xdg-open'
command = [path]
run_in_path = None
internal = True
else:
# This is a command that should be run, get run_in_path
run_in_path = (
_get_run_directory_from_possible_paths(
[(0, path)]
)
)[2]
# Check for supported file types
if last_file_arg is not None:
if last_file_arg.endswith('.cpl'):
if not first_command.endswith('control'):
command = ['control'] + command[1:]
name = "Control Panel"
internal = True
elif last_file_arg.endswith('.lnk'):
# This is a Windows shell link, read the target and open that directly
try:
#link = parsers.read_lnk_file(util.wintounix(last_file_arg))
link = binary.windows_link(util.wintounix(last_file_arg))
except:
link = False
if link:
# Convert link to a unix path and back to deal with Wine's
# inability to handle Windows "wildcards" (the tilde character)
link_win = util.unixtowin(util.wintounix(link['location']))
link_unix = util.wintounix(link_win)
if 'work dir' in link:
run_in_path_override = util.wintounix(link['work dir'])
# If the file type is not handled by Wine normally,
# then open it with xdg-open
if (
os.path.exists(link_unix) and
util.file_get_mimetype(link_unix) not in common.WINDOWS_FORMATS
):
print("Not normal Windows format, using xdg-open instead: ", util.file_get_mimetype(link_unix))
executable = 'xdg-open'
command = [link_unix]
run_in_path = None
internal = True
else:
# Change the .lnk filename to the one it's pointing to
command[last_file_arg_index] = link_win
remaining_args = [
i.lower() for i in command
if (
i != link_win and
not i.lower().endswith('start') and
not i.lower().endswith('start.exe')
)
]
# This is just a call to a normal file, we can handle that
if remaining_args == ['/unix']:
if 'work dir' in link:
parsed_command = parse_command(
[link_win],
env,
run_in_path = util.wintounix(link['work dir'])
)
else:
parsed_command = parse_command(
[link_win],
env
)
return parsed_command
elif last_file_arg.endswith('.msi'):
# This is a Windows Installer, run in and monitor it
command = ['msiexec', '/i'] + command
name = "Installer"
elif last_file_arg.endswith('.url'):
if last_file_arg[1:].startswith(':\\'):
url_filename = util.wintounix(last_file_arg)
with open(url_filename, 'r') as url_fileobj:
url = url_fileobj.read()
url = re.search(r'(?im)\[InternetShortcut\]\s+URL=(.*?)\s+?$', url)
if url:
#print("It checks out, run it.")
# This is a URL, open it in a web browser, of course without monitoring it
url = url.groups()[0]
executable = 'xdg-open'
command = [url]
run_in_path = None
internal = True
for index, argument in enumerate(command):
command[index] = util.enhance_windows_path(
command[index]
)
if run_in_path_override:
run_in_path = run_in_path_override
print("Run in", run_in_path)
print("Env (changes):", util.dict_diff(os.environ, env))
print({
'command': executable,
'arguments': command,
'internal': internal,
'prefix': wine_prefix,
'env': env,
'path': run_in_path,
'name': name
# Debug variables
,'_possible_paths': possible_paths
,'_last_file': last_file_arg
})
return {
'command': executable,
'arguments': command,
'internal': internal,
'prefix': wine_prefix,
'env': env,
'path': run_in_path,
'name': name
# Debug variables
,'_possible_paths': possible_paths
,'_last_file': last_file_arg
}
def main():
ret = util.path_exists('C:/tmp/a')
print('OK: ' + str(ret))
def main(parser, options, args):
"""
Main commnand line entry point.
"""
verbosity = options.verbosity
action = options.action
copy_files_path = options.copy_files
license_text_path = options.license_text_location
mapping_config = options.mapping
extract_license = options.extract_license
action_num = 0
api_url = ''
api_key = ''
gen_license = False
dje_license_dict = {}
if options.version:
print(__full_info__)
sys.exit(0)
if verbosity == 1:
handler.setLevel(logging.ERROR)
elif verbosity >= 2:
handler.setLevel(logging.WARNING)
valid_actions = 0, 1, 2, 3
if action:
if action in valid_actions:
action_num = action
else:
print('ERROR: Invalid action: must be one of: 0, 1, 2 or 3')
sys.exit(errno.EINVAL)
if copy_files_path:
# code to handle tilde character
copy_files_path = os.path.abspath(expanduser(copy_files_path))
if not path_exists(copy_files_path):
print("ERROR: The COPY_FILES path does not exist.")
sys.exit(errno.EINVAL)
if license_text_path:
# code to handle tilde character
license_text_path = os.path.abspath(expanduser(license_text_path))
if not path_exists(license_text_path):
print("ERROR: LICENSE_TEXT_LOCATION path does not exist.")
sys.exit(errno.EINVAL)
mapping_config_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'MAPPING.CONFIG')
if mapping_config and not path_exists(mapping_config_path):
print("ERROR: The file 'MAPPING.CONFIG' does not exist.")
sys.exit(errno.EINVAL)
if extract_license:
api_url = extract_license[0].partition('--api_url=')[2]
api_key = extract_license[1].partition('--api_key=')[2]
gen_license = True
if not len(args) == 2:
print('ERROR: <input_path> and <output_path> are required.')
print()
parser.print_help()
sys.exit(errno.EEXIST)
input_path, output_path = args
output_path = abspath(output_path)
if not output_path.endswith('/'):
output_path += '/'
if not exists(input_path):
print('ERROR: <input_path> file does not exist.')
print()
parser.print_help()
sys.exit(errno.EEXIST)
if not exists(output_path):
print('ERROR: <output_path> directory does not exist.')
print()
parser.print_help()
sys.exit(errno.EEXIST)
if not isdir(output_path):
print('ERROR: <output_path> must be a directory, not a file.')
print()
parser.print_help()
sys.exit(errno.EISDIR)
if not input_path.endswith('.csv'):
print("ERROR: <input_path> file must be a CSV file ends with '.csv'")
print()
parser.print_help()
sys.exit(errno.EINVAL)
gen = GenAbout()
dup_keys = gen.get_duplicated_keys(input_path)
if dup_keys:
print('ERROR: The <input_path> CSV file contains duplicated column keys. '
'Duplicated column keys are not allowed.')
print(dup_keys)
print()
print('Please fix the input file and re-run the tool.')
sys.exit(errno.EINVAL)
# Remove the previous log file if exist
log_path = join(output_path, LOG_FILENAME)
if exists(log_path):
os.remove(log_path)
file_handler = logging.FileHandler(log_path)
file_logger.addHandler(file_handler)
input_data = load_data_from_csv(input_path)
input_data = filter_dicts_of_empty_values(input_data)
user_keys = []
if mapping_config:
mappings = get_mappings(location=None)
input_data = apply_mappings(input_data, mappings)
user_keys = mappings.keys()
gen.validate(input_data)
ignored_fields_list = get_unknown_fields(input_data, user_keys)
if ignored_fields_list:
input_list = gen.get_only_supported_fields(input_data, ignored_fields_list)
else:
input_list = input_data
if copy_files_path:
if not isdir(copy_files_path):
print("WARNING: The COPY_FILES path must be a directory.")
print("'--copy_files' option ignored.")
else:
licenses_in_project = True
license_list = gen.verify_files_existence(input_list,
copy_files_path,
licenses_in_project)
if not license_list:
print("WARNING: No file found. '--copy_files' is ignored.")
else:
gen.copy_files(output_path, license_list)
if license_text_path:
if not isdir(license_text_path):
print("WARNING: The LICENSE_TEXT_LOCATION path must be a directory.")
print("'--license_text_location' option ignored.")
else:
licenses_in_project = False
license_list = gen.verify_files_existence(input_list,
license_text_path,
licenses_in_project)
if not license_list:
print("WARNING: No file found. '--copy_files' option ignored.")
else:
gen.copy_files(output_path, license_list)
if extract_license:
if not api_url or not api_key:
print("Missing argument for --extract_license")
sys.exit(errno.EINVAL)
for line in input_list:
try:
if line['dje_license_key']:
break
except Exception as e:
print(repr(e))
print("ERROR: The <input_path> CSV does not contain the required column 'dje_license_key' ")
sys.exit(errno.EINVAL)
if gen_license:
# Strip the ' and " for api_url, and api_key from input
api_url = api_url.strip("'").strip("\"")
api_key = api_key.strip("'").strip("\"")
dje_license_dict = gen.pre_process_and_dje_license_dict(input_list, api_url, api_key)
dje_license_list = gen.get_dje_license_list(output_path, input_list, gen_license, dje_license_dict)
components_list = gen.pre_generation(output_path, input_list, action_num)
formatted_output = gen.format_output(components_list)
gen.write_output(formatted_output)
if dje_license_list:
license_list_context = gen.process_dje_licenses(dje_license_list, dje_license_dict, output_path)
gen.write_licenses(license_list_context)
print('Completed.')
gen.warnings_errors_summary()
print('Warnings: %s' % len(gen.warnings))
print('Errors: %s' % len(gen.errors))
def pre_generation(self, gen_location, input_list, action_num):
"""
Perform some pre-generation.
TODO: document me
"""
output_list = []
for line in input_list:
component_list = []
file_location = line['about_file']
# TODO: The following few line of code seems to change the value
# without checking the action num which is incorrect.
# Get the filename from the file_location and put it as the
# value for 'about_resource'
if file_location.startswith('/'):
file_location = file_location.partition('/')[2]
if not file_location.endswith('.ABOUT'):
if file_location.endswith('/'):
file_location = dirname(file_location)
file_location = join(file_location, basename(file_location))
file_location += '.ABOUT'
# The following code is to check if there is any directory ends with spaces
split_path = file_location.split('/')
dir_endswith_space = False
for segment in split_path:
if segment.endswith(' '):
msg = 'File path contains directory name ends with spaces which is not allowed. Generation skipped.'
self.errors.append(Error(VALUE, 'about_file_path',
file_location, msg))
dir_endswith_space = True
break
if dir_endswith_space:
continue
about_file_location = join(gen_location, file_location)
about_file_dir = dirname(about_file_location)
if not os.path.exists(about_file_dir):
# Check for invalid file path
try:
makedirs(about_file_dir)
except:
msg = 'Invalid ABOUT file path.'
self.errors.append(Error(VALUE, 'about_file_path',
about_file_dir, msg))
continue
about_file_exist = path_exists(about_file_location)
if about_file_exist:
if action_num == ACTION_DO_NOTHING_IF_ABOUT_FILE_EXIST:
msg = 'ABOUT file already existed. Generation is skipped.'
self.warnings.append(Warn(IGNORED, 'about_file',
about_file_location, msg))
continue
# Overwrites the current ABOUT field value if it existed
elif action_num == ACTION_OVERWRITES_THE_CURRENT_ABOUT_FIELD_VALUE_IF_EXIST:
about_object = AboutFile(about_file_location)
for field_name, value in about_object.parsed.items():
field_name = field_name.lower()
if not field_name in line.keys() or not line[field_name]:
line[field_name] = value
# Keep the current field value and only add the "new" field
# and field value
elif action_num == ACTION_KEEP_CURRENT_FIELDS_UNCHANGED_AND_ONLY_ADD_NEW_FIELDS:
about_object = AboutFile(about_file_location)
for field_name, value in about_object.parsed.items():
field_name = field_name.lower()
line[field_name] = value
# We do not need to do anything if action_num is 3 as the
# original ABOUT file will be replaced in the write_output()
elif action_num == ACTION_REPLACE_THE_ABOUT_FILE_WITH_THE_CURRENT_GENERATED_FILE:
pass
# The following is to ensure about_resource is present.
# If they exist already, the code will not changes these.
self.update_about_resource(line, about_file_exist)
component_list.append(about_file_location)
component_list.append(line)
output_list.append(component_list)
return output_list