def __init__(self, property_file_name):
self.dataset_info = None
self.dataset_info_tsv_path = None
self.prop_file_name = property_file_name
if not os.path.isfile(property_file_name):
raise Exception("property file does not exist: " + property_file_name)
#Open the properties file
propMgr = Property()
self.props = propMgr.load_property_files(property_file_name)
self.data_root_path = file_helper.ensureTrailingSlash(self.get_prop('root.path.to.data'))
self.ingest_api_url = file_helper.ensureTrailingSlashURL(self.get_prop("ingest.api.url"))
self.nexus_token = self.get_prop("nexus.token").strip()
self.entity_api_url = file_helper.ensureTrailingSlashURL(self.get_prop("entity.api.url"))
self.uuid_api_url = file_helper.ensureTrailingSlashURL(self.get_prop("uuid.api.url"))
self.dataset_info_tsv_path = self.get_prop("vand.dataset.info.tsv")
if string_helper.isBlank(self.dataset_info_tsv_path) or not os.path.isfile(self.dataset_info_tsv_path):
raise Exception("dataset info file does not exist:" + self.dataset_info_tsv_path)
if not self.dataset_info_tsv_path.endswith(".tsv"):
raise Exception("dataset info file must be of type .tsv : " + self.dataset_info_tsv_path)
self.dataset_info = []
with open(self.dataset_info_tsv_path, newline='') as tsvfile:
reader = csv.DictReader(tsvfile, delimiter='\t')
for row in reader:
info_row = {}
for key in row.keys():
info_row[key] = row[key]
self.dataset_info.append(info_row)
self.collections = {}
self.meta_info = None
def __init__(self, path):
prop = Property()
config = prop.load_property_files(path)
self.log_level = config['log_level']
self.access_token = config['access_token']
self.output_size_mb = int(config['output_size_mb'])
self.tickers_path = config['tickers_path']
self.dropbox_folder_upload_monthly = config[
'dropbox_folder_upload_monthly']
self.dropbox_folder_upload_daily = config[
'dropbox_folder_upload_daily']
self.dropbox_chunck = int(config['dropbox_chunck'])
self.dropbox_timeout = float(config['dropbox_timeout'])
self.tickers_folder = config['tickers_folder']
self.data_folder_monthly = config['data_folder_monthly']
self.data_folder_daily = config['data_folder_daily']
self.data_folder_monthly_dropbox = config[
'data_folder_monthly_dropbox']
self.data_folder_daily_dropbox = config['data_folder_daily_dropbox']
self.result_folder_monthly = config['result_folder_monthly']
self.result_folder_daily = config['result_folder_daily']
self.tmp_folder_monthly = config['tmp_folder_monthly']
self.tmp_folder_daily = config['tmp_folder_daily']
self.encoding = config['encoding']
self.tz = int(config['tz'])
self.categories = config['categories'].split(',')
self.year_from = config['year_from']
self.year_until = config['year_until']
self.geo = config['geo']
self.retries = int(config['retries'])
self.backoff_factor = int(config['backoff_factor'])
self.gtrends_timeout_connect = int(config['gtrends_timeout_connect'])
self.gtrends_timeout_read = int(config['gtrends_timeout_read'])
self.prefix = config['prefix']
def _get_apk_md5_from_properties(fp):
prop = Property()
dic_prop = prop.load_property_files(fp)
if APP_APK_MD5 in dic_prop:
return dic_prop[APP_APK_MD5]
else:
return None
def __init_properties(self, file_name):
self.properties_file_name = file_name
if not os.path.isfile(file_name):
raise Exception("Property file " + file_name + " is required and was not found.")
propMgr = Property()
self.props = propMgr.load_property_files(file_name)
self.dyn_hostname = self.__getProperty("hostname")
self.access_key = self.__getProperty("aws.access.key")
self.access_secret = self.__getProperty("aws.access.secret")
def __init__(self):
prop = Property()
self.propertyFile = sys.argv[1] if len(sys.argv) > 1 else ''
logging.debug(self.propertyFile)
if not (os.path.exists(self.propertyFile)): #if file does not exist
raise OSError(
"Property file not found! Please provide property file as an argument."
)
#Load the property file
self.dic_prop = prop.load_property_files(self.propertyFile)
def __init__(self, property_file_name, assay_row_key):
#Open the properties file
propMgr = Property()
self.prop_file_name = property_file_name
self.assay_row_key = assay_row_key
if not os.path.isfile(self.prop_file_name):
raise Exception("Required property file does not exist: " + self.prop_file_name)
self.props = propMgr.load_property_files(self.prop_file_name)
root_path = file_helper.ensureTrailingSlashURL(self.get_prop("root.path.to.data"))
clinical_meta_path = self.join_dir(root_path, "metadata/clinical")
assay_meta_path = self.join_dir(root_path, "metadata/assay")
self.clinical_metadata = self.convert_clin_meta_to_dict(clinical_meta_path)
self.assay_metadata = self.convert_assay_meta_to_dict(assay_meta_path)
def __init__(self, path):
prop = Property()
config = prop.load_property_files(path)
self.log_level = config['log_level']
self.access_token = config['access_token']
self.threshold = int(config['threshold'])
self.error_code_limit = int(config['error_code_limit'])
self.output_size_mb = int(config['output_size_mb'])
self.dropbox_folder = config['dropbox_folder']
self.dropbox_chunck = int(config['dropbox_chunck'])
self.dropbox_timeout = float(config['dropbox_timeout'])
self.data_path = config['data_path']
self.master_path = config['master_path']
self.results_path = config['results_path']
self.since_year = int(config['since_year'])
def __init__(self, path):
prop = Property()
config = prop.load_property_files(path)
self.log_level = config['log_level']
self.access_token = config['access_token']
self.output_size_mb = int(config['output_size_mb'])
self.dropbox_folder_download = config['dropbox_folder_download']
self.dropbox_folder_upload = config['dropbox_folder_upload']
self.dropbox_chunck = int(config['dropbox_chunck'])
self.dropbox_timeout = float(config['dropbox_timeout'])
self.data_folder = config['data_folder']
self.result_folder = config['result_folder']
self.file_regex = config['file_regex']
self.result_prefix = config['result_prefix']
self.result_extension = config['result_extension']
self.encoding_input = config['encoding_input']
def setUp(self):
self.log.debug("Properties initialization")
self.prop = Property()
try:
self.dic_prop = self.prop.load_property_files("/Users/user/PycharmProjects/BGETest/config/config.properties")
self.log.debug("Properties successfully loaded.")
self.log.debug(self.dic_prop)
except FileNotFoundError as err:
self.log.error("config.properties file not found!!!")
raise ValueError("There is no config.properties file in the project directory" + str(err))
browserName = self.dic_prop.get("browser", "chrome")
if browserName.lower() == "firefox":
self.driver = webdriver.Firefox()
self.log.debug("Object for firefox is created - " + str(self.driver))
elif browserName.lower() == "chrome":
self.driver = webdriver.Chrome()
self.log.debug("Object for chrome is created - " + str(self.driver))
elif browserName.lower() == "safari":
self.driver = webdriver.Safari()
self.log.debug("Object for safari is created - " + str(self.driver))
else:
raise ValueError("Unknown browser, please check the config.properties file")
self.eventListener = WebEventListener()
self.driver = EventFiringWebDriver(self.driver, self.eventListener)
self.driver.maximize_window()
self.driver.delete_all_cookies()
self.log.debug("Cookies are successfully deleted")
self.driver.set_page_load_timeout(self.PAGE_LOAD_TIME)
self.driver.implicitly_wait(self.IMPLICITLY_WAIT)
self.driver.get(self.dic_prop.get("url"))
self.longMessage = False
self.log.debug(self.id())
self.screenshot = ScreenShot(self.driver)
try:
self.screenshotName = "./results/{}.png".format(str(self.id()).split('.')[3])
except IndexError:
self.screenshotName = "./results/{}.png".format(str(self.id()).split('.')[2])
def __init__(self,
property_file_name,
argparse_args=None,
required_props=[]):
self.prop_file_name = property_file_name
if argparse_args is None and not os.path.isfile(property_file_name):
raise ErrorMessage("property file does not exist: " +
property_file_name)
if property_file_name is None:
if argparse_args is None:
raise ErrorMessage(
"property file name or argparse arguments required")
self.props = {}
for arg_key in argparse_args.keys():
val = argparse_args[arg_key]
if not val is None:
if isinstance(val, list):
if len(val) == 1:
self.props[arg_key] = val[0]
elif len(val) > 1:
self.props[arg_key] = val
else:
propMgr = Property()
self.props = propMgr.load_property_files(property_file_name)
not_found = []
for required_prop in required_props:
if required_prop not in self.props or string_helper.isBlank(
self.props[required_prop]):
not_found.append(required_prop)
if len(not_found) > 0:
if len(not_found) == 1:
prop = "property"
else:
prop = "properties"
raise ErrorMessage("Required " + prop + " not found in " +
property_file_name + ":[" +
string_helper.listToCommaSeparated(not_found) +
"]")
def get_client_config():
"""
https://www.itiger.com/openapi/info 开发者信息获取
:return:
"""
# Load tiger account information
prop = Property()
prop_loader = prop.load_property_files(
os.path.expanduser('~/config/env.properties'))
tiger_id = prop_loader.get('tiger_id')
tiger_account = prop_loader.get('tiger_account')
is_sandbox = False
client_config = TigerOpenClientConfig(sandbox_debug=is_sandbox)
client_config.private_key = read_private_key(
os.path.expanduser('~/.ssh/tigerbroker_rsa_private_key.pem'))
client_config.tiger_id = tiger_id
client_config.account = tiger_account
client_config.language = Language.en_US
return client_config
def __init__(self, prop_file_name):
self._prop_file_name = prop_file_name
#check to make sure we have a properties file
if not os.path.isfile(prop_file_name):
raise Exception("Property file " + prop_file_name +
" is required and does not exist.")
#Open the properties file
propMgr = Property()
self.props = propMgr.load_property_files(prop_file_name)
self.entity_api_url = file_helper.ensureTrailingSlashURL(
self.get_prop("entity.api.url"))
self.nexus_token = self.get_prop("nexus.token").strip()
self.globus_system_dir = self.get_prop("globus.system.dir").strip()
self.globus_base_url = file_helper.removeTrailingSlashURL(
self.get_prop("globus.single.file.base.url").strip())
self.test_group_id = self.get_prop("test.group.id").strip().lower()
self.globus_app_base_url = self.get_prop("globus.app.base.url").strip()
def __init__(self, property_file_name, required_props=[]):
self.prop_file_name = property_file_name
if not os.path.isfile(property_file_name):
raise ErrorMessage("property file does not exist: " +
property_file_name)
propMgr = Property()
self.props = propMgr.load_property_files(property_file_name)
not_found = []
for required_prop in required_props:
if required_prop not in self.props or string_helper.isBlank(
self.props[required_prop]):
not_found.append(required_prop)
if len(not_found) > 0:
if len(not_found) == 1:
prop = "property"
else:
prop = "properties"
raise ErrorMessage("Required " + prop + " not found in " +
property_file_name + ":[" +
string_helper.listToCommaSeparated(not_found) +
"]")
def get_prop(prop_name):
if not prop_name in props:
raise Exception("Required property " + prop_name + " not found in " +
prop_file_name)
val = props[prop_name]
if string_helper.isBlank(val):
raise Exception("Required property " + prop_name + " from " +
prop_file_name + " is blank")
return props[prop_name]
#check to make sure we have a properties file
if not os.path.isfile(prop_file_name):
raise Exception("Property file " + prop_file_name +
" is required and does not exist.")
#Open the properties file
propMgr = Property()
props = propMgr.load_property_files(prop_file_name)
root_path = file_helper.ensureTrailingSlashURL(get_prop("root.path.to.data"))
uuid_api_url = file_helper.ensureTrailingSlashURL(get_prop("uuid.api.url"))
nexus_token = get_prop("nexus.token").strip()
dirs = os.listdir(root_path)
printHeader()
for name in dirs:
if not name.startswith("LC "):
handleMultDatasetDir(os.path.join(root_path, name))
print(name)
def __init__(self):
self.prop_config = Property()
self.prop = self.prop_config.load_property_files('./config.properties')
# def csv(self, header=None, data_columns=('{{name}}', '{{address}}'), num_rows=10, include_row_ids=False):
"""
write a mapping piece for mapping semantic cols from business tables to equivalent faker types
e.g pool number should map to int(8) or ########
pool issuercode to XX####
ppol type - #, range => (1,3)
pool status - #, range => (3,3)
approval date
approval amount
"""
p = Property()
props = p.load_property_files('pool.schema')
# p.load(open('pool.schema'))
print(props)
print("===")
header = []
for key in props.keys():
val = props.get(key)
# print(key+"::"+val)
header.append(key)
# print(f"in {key} the val is {val}; found format at {val.find('format')}")
if val.find('format') >= 0:
f = val.split(":")
# print("found format :: " + str(f))
def getProperty(self, name):
prop = Property()
dic_prop = prop.load_property_files('config.properties')
return dic_prop.get(name)
def ingestPrivateData(propertiesPath):
input_file = propertiesPath
print('1- We start the ingestion process')
print('2- The content of the file ' + input_file + ' is read')
allVariableAreFilled = True
if Path(input_file).is_file():
# load properties file
data_criteria = Property().load_property_files(input_file)
# add the criteria from the properties file to the dictionary
criteria_list = {}
try:
user_id = data_criteria['user_id']
if len(user_id) > 0:
criteria_list['userId'] = user_id
print(' 2.1- The user id is ' + user_id)
else:
allVariableAreFilled = False
print(' 2.1- The user is mandatory')
except:
pass
try:
data_path = data_criteria['data_path']
if len(data_path) > 0:
criteria_list['dataPath'] = data_path
print(' 2.2- The data path is ' + data_path)
else:
allVariableAreFilled = False
print(' 2.2- The data path is mandatory')
except:
pass
try:
data_format = data_criteria['data_format']
if len(data_format) > 0:
criteria_list['dataFormat'] = data_format
print(' 2.3- The data format is ' + data_format)
else:
allVariableAreFilled = False
print(' 2.3- The data format is mandatory')
except:
pass
# location = data_criteria['location']
try:
polarization = data_criteria['polarization']
if len(polarization) > 0:
criteria_list['polarization'] = polarization
print(' 2.4- The polarization is ' + polarization)
else:
allVariableAreFilled = False
print(' 2.4- The polarization is mandatory')
except:
pass
try:
sub_region_name = data_criteria['sub_region_name']
if len(sub_region_name) > 0:
criteria_list['subregionName'] = sub_region_name
print(' 2.5- The sub region is ' + sub_region_name)
else:
allVariableAreFilled = False
print(' 2.5- The sub region is mandatory')
except:
pass
try:
collection_name = data_criteria['collection_name']
if len(collection_name) > 0:
criteria_list['collectionName'] = collection_name
print(' 2.4- The collection is ' + collection_name)
else:
allVariableAreFilled = False
print(' 2.4- The collection is mandatory')
except:
pass
else:
print('ERROR: The file "' + input_file + '" does not exist.')
logging.error('The file "' + input_file + '" does not exist.')
if (allVariableAreFilled):
print(
"All variables in sharedata.properties are filled. We start the process"
)
##We move the data in the /data/private using the user id
print("3- Starting the copy of the file to a temp folder")
folder_destination = "/app/User_data/"
folder_destination = createIfnotExistFolder(folder_destination,
criteria_list['userId'])
if (criteria_list['dataFormat']) == 'ROI':
print(
"3.1- File format detected is ROI. We need 4 files : dbf, shp, shx, prj"
)
listeOfRoi = isROIfileExistAll(data_path)
if (len(listeOfRoi) > 0):
#We move the 4 ROI files
for i in range(len(listeOfRoi)):
filePath = listeOfRoi[i]
shutil.copy(filePath, folder_destination)
print("4- Files correctly copied")
else:
print(
"4-we cannot ingest this ROI files. Some files are missing"
)
exit()
else:
shutil.copy(data_path, folder_destination)
print("4- File correctly copied")
######We call the back end service to start the ingestion
ingestTheData(
criteria_list, folder_destination + "/" +
os.path.basename(criteria_list['dataPath']))
else:
print("Please, fill all the mandatories variables !!!!!!!!")
def test_config(self, key):
prop = Property()
filepath = prop.load_property_files(configPath)
prop2 = filepath.__getitem__(key)
return prop2
def getPropertyFile(self, key):
prop = Property()
disc_prop = prop.load_property_files(
"F:\selenium\PageObjectModule\config.properties")
value = disc_prop.get(key)
return value