def test3():
data = [numpy.arange(10) + 0.1, numpy.arange(10) + 0.1]
names = ["latitudes", "longitudes"]
units = ["degrees_north", "degrees_east"]
missing = [-9999., -9999.]
types = ["f4", "f4"]
field_object = fields.Fields(data, names, types, units, missing)
WxtCdf("out.nc", field_object)
def test2():
data = [numpy.arange(10) + 0.1]
names = ["data"]
types = ["f4"]
units = ["degrees_north"]
missing = [-9999.]
field_object = fields.Fields(data, names, types, units, missing)
WxtCdf("out.nc", field_object)
def test1():
data = [numpy.arange(10)]
names = ["data"]
missing = [-9999]
types = ["i4"]
units = [""]
field_object = fields.Fields(data, names, types, units, missing)
WxtCdf("out.nc", field_object)
def test4():
data = [numpy.arange(10) + 0.1, numpy.arange(10) + 0.1, numpy.arange(10) + 0.1, numpy.arange(10) + 0.1, numpy.arange(10) + 0.1]
names = ["obs_time", "rec_time", "latitude", "longitude", "elevation"]
missing = [-9999., -9999., -9999., -9999., -9999.]
types = ["f8", "f8", "f8", "f8", "f4"]
units = ["seconds since 1970-1-1 00:00", "seconds since 1970-1-1 00:00", "degrees_north", "degrees_east", "m"]
sizes = [10, 10, 10, 10, 10]
field_object = fields.Fields(data, names, types, units, missing, sizes)
WxtCdf("out.nc", field_object)
def parse(in_file, out_file, cdfout, logg):
"""
Takes a single json input file, converts it to netcdf or ASCII, and writes it out
"""
files_written = []
field_names = [
"id", "unit_id", "insertion_time", "observation_time", "latitude",
"longitude", "elevation", "air_temp", "atmospheric_pressure",
"light_level", "rain_intensity", "relative_humidity", "road_temp"
]
field_types = [
"a32", "a32", "a32", "a32", "f4", "f4", "f4", "f4", "f4", "f4", "f4",
"f4", "f4"
]
field_units = [
None, None, "seconds since 1970-01-01 00 UTC",
"seconds since 1970-01-01 00 UTC", "degrees north", "degrees east",
"meters", "celsius", "hPa", "lux", None, "percent", "celsius"
]
data = []
missing = [
"", "", "", "", -9999., -9999., -9999., -9999., -9999., -9999., -9999.,
-9999., -9999.
]
sizes = [
None, None, None, None, None, None, None, None, None, None, None, None,
None
]
field_object = fields.Fields(data, field_names, field_types, field_units,
missing, sizes)
logg.write_info("parsing input file %s" % in_file)
wxt_parse = wxtParse(in_file)
observation_sets = ObservationSets("observation_time", field_object,
wxt_parse.observation_set_dict)
if cdfout:
array = observation_sets.get_field_data()
data_fields = []
for field_name in field_names:
data_fields.append(array[field_name])
field_object.data = data_fields
logg.write_info("writing netcdf file %s" % out_file)
files_written.append(out_file)
wxt_cdf.WxtCdf(out_file, field_object)
else:
fp_out = file(out_file, "w")
files_written.append(out_file)
observation_sets.write_ascii(fp_out)
fp_out.close()
return files_written
def stocks(T, portfolio, portfolio_is_complete):
"""
DataFrame Update 4 - Addtional columns added here from Stock Object calls or other APIs
"""
feather = featherfile('stocks')
cache_exists = os.path.exists(feather)
S = None
if cache_exists:
S = pd.read_parquet(feather)
# TODO: This needs to be disabled when Fields is updated
if portfolio_is_complete: return S
data = _pull_processed_holdings_data(portfolio)
# 8. Field Extensions
flds = fields.Fields(data)
S = flds.refreshed(S)
extensions = flds.extensions(S, T)
with ShadyBar('%48s' % 'Refreshing Extended DataFrame and Cache',
max=len(extensions) + 2) as bar:
for field in extensions:
flds.extend(field, S, T)
bar.next()
if 'test' not in feather:
# Emergency Sanitization
# for column in S.columns:
# S[column] = S[column].map(lambda n: F(n) if type(n) in (str, float, int) else n)
try:
S.to_parquet(feather, index=False)
except:
print("Amended DataFrame can't be dumped")
util.debug.ddump(
{
column: str(set(map(type, S[column])))
for column in S.columns
},
force=True)
raise
bar.next()
flds.sort(S)
bar.next()
return S
import fields
import string
import random
import re
# SHARED VARIABLES
tokens = fields.Fields("messages.xml")
metrics = dict()
features = dict()
utilities = [
"ts", "state.code", "state.level", "state.description", "client.id",
"probe.id"
]
tables = dict()
timeouts = dict()
attacks = dict()
table_clist = set()
__charset = string.letters + string.digits
class PrintUtils(object):
_regex = None
@classmethod
def replace(self, body, on_match):
if not PrintUtils._regex:
feats_tokens = '|'.join(features)
def main_menu():
global disk_volume, lun_size, execute_change
def _cancel_change(change_file_name):
orig_change = '{0}/stgadm/changes/{1}.py'.format(
config.stghome, change_file_name)
dest_change = '{0}/stgadm/changes_canceled/{1}.py'.format(
config.stghome, change_file_name)
os.rename(orig_change, dest_change)
def _finish_change(change_file_name):
orig_change = '{0}/stgadm/changes/{1}.py'.format(
config.stghome, change_file_name)
dest_change = '{0}/stgadm/changes_executed/{1}.py'.format(
config.stghome, change_file_name)
os.rename(orig_change, dest_change)
os.system('clear')
print('')
print('[ Storage Adm ]')
print('[ ]')
print('[ Version {0} - (c) 2016 Kairo Araujo ]'.format(globalvar.version))
print('[ http://github.com/kairoaraujo/STGAdm ]')
print('[ BSD License ]')
print('[ ]')
stgadm = raw_input('\nMain Menu:\n\n'
'1. Add new volumes to existent host. (create change)\n'
'2. Execute changes created.\n'
'\nPlease choose an option: ')
# Add new volumes to existent host menu
if stgadm == '1':
print('\n\n[ REQUEST INFORMATIONS ]\n')
change = fields.Fields('change', 'Ticket/Change/Work Order: ')
change.chkfieldstr()
change = change.strvarout()
hostname_client = fields.Fields('hostname', 'Hostname Client: ')
hostname_client.chkfieldstr()
hostname_client = hostname_client.strvarout()
wwn_client = fields.Fields('wwn_client', 'WWN Server Client: ')
wwn_client.chkfieldstr()
wwn_client = wwn_client.strvarout()
disk_volume = int()
lun_size = int()
while True:
try:
disk_volume = int(raw_input("Total Disk Required (GB): "))
break
except (TypeError, ValueError):
print('\tERROR: Total Disk need to be an int value in GB. '
'\nDo not use GB. Example: 1000 for 1000GB (1TB).')
while True:
try:
lun_size = int(raw_input("Default LUN size (GB): "))
if disk_volume % lun_size == 0:
break
else:
print('\tERROR: The LUN size need to be a divisor of Total'
'required disk.'
'\n\t If Total Disk Required is wrong, please '
'use "Ctrl+C".')
except (TypeError, ValueError):
print('\tERROR: LUN Size need to be an int value.'
'\nDo not use GB. Example 100 for 100GB size of LUNs.')
if ':' in wwn_client:
wwn_client = wwn_client.replace(':', '')
# select storage
get_stg = systemstorages.SystemStorages()
get_stg.selectstorage()
stg_type = get_stg.gettype()
if stg_type == 'EMC_VMAX':
stg_sid = get_stg.getsid()
stg_name = get_stg.getstorage()
storage_name = stg_name
menu_emc_vmax(change, hostname_client, storage_name, stg_name,
stg_type, stg_sid, wwn_client)
elif stg_type == 'IBM_DS8K':
stg_sid = get_stg.getsid()
stg_name = get_stg.getstorage()
storage_name = stg_name
menu_ibm_ds8k(change, hostname_client, storage_name, stg_name,
stg_type, stg_sid, wwn_client)
elif stg_type == 'EMC_VNX':
storage_name = get_stg.getstorage()
stg_1ip = get_stg.getvnx_1ip()
stg_2ip = get_stg.getvnx_2ip()
stg_user = get_stg.getvnx_user()
stg_pass = get_stg.getvnx_pass()
stg_scope = get_stg.getvnx_scope()
menu_emc_vnx(change, hostname_client, storage_name, storage_name,
stg_type, stg_1ip, stg_2ip, stg_user, stg_pass,
stg_scope, wwn_client)
else:
print('ERROR: Storage type {0} invalid. Check config file'.format(
stg_type))
elif stgadm == '2':
change_file = findchange.select().replace('.py', '')
# import change_file
change_module = importlib.import_module(
'stgadm.changes.{0}'.format(change_file))
try:
change_module.preview()
except ValueError:
if change_module.stg_type == 'IBM_DS8K':
print('Removing LUN ID reserve...')
for l_1_id in change_module.lss_1_id_list:
ds8k_add_dev.remove_reserved_id(l_1_id)
for l_2_id in change_module.lss_2_id_list:
ds8k_add_dev.remove_reserved_id(l_2_id)
print('LUN ID cleanup finished.')
elif change_module.stg_type == 'EMC_VNX':
print('Removing LUN ID reserve...')
# remove lu:
for lu_id in change_module.lu_ids:
vnx_add_dev.remove_reserved_id(lu_id)
print('Removing HLU ID reserve... ')
# remove hlu
for hlu_id in change_module.hlu_ids:
vnx_add_dev.remove_reserved_hlu_id(
hlu_id, change_module.stggroup_name)
# remove hlu if is for cluster
for cluster_node in change_module.cls_nodes.keys():
for hlu_id in change_module.cls_nodes[cluster_node][3]:
vnx_add_dev.remove_reserved_hlu_id(
hlu_id, change_module.cls_nodes[cluster_node][1])
print('LUN and HLU ID cleanup finished.')
_cancel_change(change_file)
exit()
execute_change = fields.YesNo(
'Do you would like execute this '
'change?[y/n]: ', 'n')
execute_change = execute_change.check()
if execute_change == 'y':
print("Executing the change. Please wait...\n")
# import change_file
change_module = importlib.import_module(
'stgadm.changes.{0}'.format(change_file))
try:
change_module.execute()
except ValueError as e:
print("ERROR: {}".format(e))
_finish_change(change_file)
else:
print('Wrong option. Exiting.')
def menu_ibm_ds8k(change=None,
hostname_client=None,
storage_name=None,
stg_name=None,
stg_type=None,
stg_sid=None,
wwn_client=None):
global pool_1_option, pool_2_option, code_1_pool, code_2_pool
if not os.path.isfile(config.dscli_bin):
print("\nERROR: DSCLI not found. Check config file item dscli_bin.")
exit()
if not os.path.isfile(config.dscli_profile_path + '/' + stg_sid):
print("\nERROR: DSCLI not found. Check config file item "
"dscli_profile_path for {0}".format(stg_name))
exit()
ds8k = pystorage.IBM.DS8K(config.dscli_bin,
config.dscli_profile_path + '/' + stg_sid)
def _get_stg_host_info(subtitle, wwn):
volume_group = ds8k.get_volgrpid(wwn)
hostname_client_storage = ds8k.get_hostname(wwn)
if (volume_group[0] != 0) or hostname_client_storage[0] != 0:
print("{0} - {1}".format(volume_group[1],
hostname_client_storage[1]))
exit(1)
else:
print('{0}Client informations:'.format(subtitle))
print('Volume Group from \033[1;32m{0}\033[1;00m identified '
'as \033[1;32m{1}\033[1;00m.'.format(wwn_client,
volume_group[1]))
print('Hostname used on storage is '
'\033[1;32m{0}\033[1;00m\n'.format(
hostname_client_storage[1]))
return hostname_client_storage, volume_group
print('\nGetting the Volume Group and hostname used on storage ...'
'\n\nPlease wait...')
hostname_client_stg, vol_group = _get_stg_host_info('', wwn_client)
# get cluster node information.
cls = fields.YesNo('Is it a CLUSTER provisioning? [y/n]: ', 'n')
cls = cls.check()
cls_nodes = {}
if cls == 'y':
count_node = 0
check_new_cls_node = True
while check_new_cls_node is True:
cls_wwn_client = fields.Fields(
'cls_wwn_client', '\n(Cluster) Insert the WWN Server Client: ')
cls_wwn_client.chkfieldstr()
cls_wwn_client = cls_wwn_client.strvarout()
print('\nPlease wait.. ')
cls_node_info = _get_stg_host_info('[Cluster Node] ',
cls_wwn_client)
cls_nodes.update({
count_node:
[cls_node_info[0][1], cls_node_info[1][1], cls_wwn_client]
})
count_node += 1
new_cls = fields.YesNo(
'Do you want add another cluster host? [y/n]: ', 'n')
new_cls = new_cls.check()
if new_cls is 'y':
check_new_cls_node = True
else:
check_new_cls_node = False
# select first and second pool
print('\n[POOLS Selection]')
print('\n\033[1;31mNOTE:\033[1;00m '
'The Primary pool will be used for extra LUN if the number '
'of LUNs is odd. '
'\nExample: Total of LUNs = 11'
'\n6 LUNs in Primary pool and 5 LUNs in the Secondary pool.\n'
'\nPlease wait...')
# pool informations
ds8k_pools = ds8k.lsextpool()
if ds8k_pools[0] != 0:
print('ERROR: {0}'.format(ds8k_pools[1]))
exit()
else:
print(ds8k_pools[1])
pool_list = []
for l_pool in ds8k_pools[1].split('\n')[3:]:
try:
pool_list.append(l_pool.split()[0])
except (IndexError, ValueError):
pass
# disk count (number of disks)
disk_count = disk_volume / lun_size
if disk_count > 1:
if (disk_count % 2) != 0:
disk_1_count = (disk_count / 2) + 1
disk_2_count = (disk_count / 2)
else:
disk_1_count = (disk_count / 2)
disk_2_count = (disk_count / 2)
else:
disk_1_count = disk_count
disk_2_count = 0
# Primary Pool
count = 0
for l_pool in pool_list:
print('{0}: {1}'.format(count, l_pool))
count += 1
while True:
try:
pool_1_option = int(raw_input("Select Primary pool: "))
break
except (IndexError, ValueError):
print('\tERROR: Select an existing option between '
'0 and {0}.'.format(count))
pool_1_option = pool_list[pool_1_option]
# cleaning the selected pool
pool_list.remove(pool_1_option)
print("\nPlease select the LUN LSS ID CODE "
"to be used by {0}\n"
"The code need to be between 00 and FF".format(pool_1_option))
while True:
try:
code_1_pool = raw_input('Digit the LSS: ')
int(code_1_pool, 16)
if len(code_1_pool) != 2:
print("The code need to be between 00 and FF.")
else:
break
except ValueError:
print("The code need to be between 00 and FF.")
print("Primary pool: {0} | LUN ID Code: {1}\n".format(
pool_1_option, code_1_pool))
# getting free IDs
lss_free = getid.DS8K(config.dscli_bin,
config.dscli_profile_path + '/' + stg_sid,
code_1_pool)
lss_1_free = lss_free.free_lss()
# primary disk
if len(lss_1_free) < disk_count:
print("ERROR: Not exist sufficient free IDs on {0} LSS.".format(
code_1_pool))
exit()
lss_1_id_list = lss_1_free[:disk_1_count]
# The second pool need to be select just for more than one disk.
if disk_count > 1:
# Secondary Pool
count = 0
for l_pool in pool_list:
print('{0}: {1}'.format(count, l_pool))
count += 1
while True:
try:
pool_2_option = int(raw_input("Select Secondary pool: "))
break
except (IndexError, ValueError):
print('\tERROR: Select an existing option between '
'0 and {0}.'.format(count))
pool_2_option = pool_list[pool_2_option]
print("\nPlease select the LUN LSS ID CODE "
"to be used by {0}\n"
"The code need to be between 00 and FF".format(pool_2_option))
while True:
try:
code_2_pool = raw_input('Digit the LSS: ')
int(code_2_pool, 16)
if len(code_2_pool) != 2:
print("The code need to be between 00 and FF.")
else:
break
except ValueError:
print("The code need to be between 00 and FF.")
print("\n* Primary pool : {0} | LUN ID Code: {1}XX\n"
"* Secondary pool : {2} | LUN ID Code: {3}XX\n".format(
pool_1_option, code_1_pool, pool_2_option, code_2_pool))
# getting free IDs for LSS Secondary
lss_free = getid.DS8K(config.dscli_bin,
config.dscli_profile_path + '/' + stg_sid,
code_2_pool)
lss_2_free = lss_free.free_lss()
lss_2_id_list = lss_2_free[:disk_2_count]
# just one disk
else:
print("\n* Primary pool: {0} | LUN ID Code: {1}XX\n".format(
pool_1_option, code_1_pool))
disk_2_count = 0
pool_2_option = None
lss_2_id_list = []
print("\nPlease wait...")
print("Please give the TAG/SID (Code to identify) the LUNs.\n"
"Use a name to identify the environment or client.\n"
"Examples: DEV or MIGRATION or FOOBAR\n\n"
"With this code the LUNs will be create as TAG/SID_LUN_SSL:\n"
"Example: DEV_LUN_A601 or MIGRATION_LUN_AA07 or FOOBAR_LUN_045A\n")
lun_sid = fields.Fields('lun_sid', 'LUN SID(TAG): ')
lun_sid.chkfieldstr()
lun_sid = lun_sid.strvarout()
new_change = ds8k_add_dev.New(
change, hostname_client, storage_name, wwn_client, stg_name, stg_type,
stg_sid, pool_1_option, pool_2_option, disk_1_count, disk_2_count,
lss_1_id_list, lss_2_id_list, disk_volume, lun_size, lun_sid,
vol_group[1], hostname_client_stg[1], disk_count, cls, cls_nodes)
new_change.preview()
save_config = fields.YesNo(
'Do you would like save this allocation?[y/n]: ', 'n')
save_config = save_config.check()
if save_config == 'y':
new_change.headerchange()
new_change.writechange()
end_change = new_change.closechange()
print(end_change)
else:
print('Finishing. Thank you.')
def menu_emc_vnx(change=None,
hostname_client=None,
storage_name=None,
stg_name=None,
stg_type=None,
stg_1ip=None,
stg_2ip=None,
stg_user=None,
stg_pass=None,
stg_scope=None,
wwn_client=None):
global pool_option, pool
if not os.path.isfile(config.naviseccli_bin):
print("\nERROR: NAVISECCLI not found. Check config file item "
"naviseccli_bin.")
exit()
disk_count = disk_volume / lun_size
vnx = pystorage.EMC.VNX(config.naviseccli_bin, stg_1ip, stg_2ip, stg_user,
stg_pass, stg_scope)
def _get_stg_host_info(subtitle, wwn):
hostname_client_stg = vnx.get_hostname(wwn)
stg_group = vnx.get_stggroup(wwn)
if (stg_group[0] != 0) or (hostname_client_stg[0] != 0):
print("{0} - {1}".format(stg_group, hostname_client_stg))
exit(1)
else:
print('{0}Client informations:'.format(subtitle))
print('Storage Group Name used by \033[1;32m{0}\033[1;00m '
'identified as \033[1;32m{1}\033[1;00m.'.format(
wwn, stg_group[1]))
print('Hostname used on storage is '
'\033[1;32m{0}\033[1;00m\n'.format(hostname_client_stg[1]))
return hostname_client_stg[1], stg_group[1]
print('\nGetting the hostname used on storage and Storage Group Name ...'
'\n\nPlease wait...')
# getting pool
pools = vnx.pool_list()
if pools[0] != 0:
print("ERROR: {0}".format(pools[1]))
exit(pools[0])
else:
if len(pools[1]) > 1:
print('\nMultiples Pools detected. Please choose a Pool.\n')
count = 0
for l_pools in pools[1]:
print('{0}: {1}'.format(count, l_pools))
count += 1
while True:
try:
pool_option = int(raw_input("Select the Pool: "))
break
except (IndexError, ValueError):
print('\tERROR: Select an existing option between'
'0 and {0}.'.format(count))
pool = pools[1][pool_option]
else:
pool = pools[1][0]
print('Using the only existent pool: '
'\033[1;32m{0}\033[1;00m\n'.format(pool))
hostname_client_storage, stggroup_name = _get_stg_host_info('', wwn_client)
# get cluster node information.
cls = fields.YesNo('Is it a CLUSTER provisioning? [y/n]: ', 'n')
cls = cls.check()
cls_nodes = {}
if cls == 'y':
count_node = 0
check_new_cls_node = True
while check_new_cls_node is True:
cls_wwn_client = fields.Fields(
'cls_wwn_client', '\n(Cluster) Insert the WWN Server Client: ')
cls_wwn_client.chkfieldstr()
cls_wwn_client = cls_wwn_client.strvarout()
print('\nPlease wait.. ')
cls_node_info = _get_stg_host_info('[Cluster Node] ',
cls_wwn_client)
# hlu id for cluster node
node_hlu_free = getid.VNX(config.naviseccli_bin, stg_1ip, stg_user,
stg_pass, stg_scope, pool)
node_hlu_free_ids = node_hlu_free.free_hlu(cls_node_info[1])
if len(node_hlu_free_ids) < disk_count:
print("ERROR: Not exist sufficient HLU free IDs "
"on {0} Storage Group.".format(cls_node_info[1]))
exit()
node_hlu_ids = node_hlu_free_ids[:disk_count]
cls_nodes.update({
count_node: [
cls_node_info[0], cls_node_info[1], cls_wwn_client,
node_hlu_ids
]
})
count_node += 1
new_cls = fields.YesNo(
'Do you want add another cluster host? [y/n]: ', 'n')
new_cls = new_cls.check()
if new_cls is 'y':
check_new_cls_node = True
else:
check_new_cls_node = False
print("Please give the TAG/SID (Code to identify) the LUNs.\n"
"Use a name to identify the environment or client.\n"
"Examples: DEV or MIGRATION or FOOBAR\n\n"
"With this code the LUNs will be create as TAG/SID_LUN_SSL:\n"
"Example: DEV_LUN_1 or MIGRATION_LUN_7 or FOOBAR_LUN_45\n")
lun_sid = fields.Fields('lun_sid', 'LUN SID(TAG): ')
lun_sid.chkfieldstr()
lun_sid = lun_sid.strvarout()
# lu id
lu_free = getid.VNX(config.naviseccli_bin, stg_1ip, stg_user, stg_pass,
stg_scope, pool)
lu_free_ids = lu_free.free_lu()
if len(lu_free_ids) < disk_count:
print("ERROR: Not exist sufficient free IDs on {0} pool.".format(pool))
exit()
lu_ids = lu_free_ids[:disk_count]
# hlu id
hlu_free = getid.VNX(config.naviseccli_bin, stg_1ip, stg_user, stg_pass,
stg_scope, pool)
hlu_free_ids = hlu_free.free_hlu(stggroup_name)
if len(hlu_free_ids) < disk_count:
print("ERROR: Not exist sufficient HLU free IDs "
"on {0} Storage Group.".format(stggroup_name))
exit()
hlu_ids = hlu_free_ids[:disk_count]
new_change = vnx_add_dev.New(change, hostname_client, storage_name,
wwn_client, stg_name, stg_type, stg_1ip,
stg_2ip, stg_user, stg_pass, stg_scope, pool,
disk_count, lu_ids, hlu_ids, disk_volume,
lun_size, lun_sid, hostname_client_storage,
stggroup_name, config.lun_type_create, cls,
cls_nodes)
new_change.preview()
save_config = fields.YesNo(
'Do you would like save this allocation?[y/n]: ', 'n')
save_config = save_config.check()
if save_config == 'y':
new_change.headerchange()
new_change.writechange()
end_change = new_change.closechange()
print(end_change)
else:
print('Finishing. Thank you.')
def main():
usage_str = "%prog in_file[s] out_file"
parser = OptionParser(usage=usage_str)
parser.add_option("-d",
"--directory",
dest="directory",
help="use all files in specified directory as input")
parser.add_option("-n",
"--netcdf_file",
dest="cdf",
action="store_true",
help="write output netcdf file")
parser.add_option("-l",
"--log",
dest="log",
help="write log messages to specified file")
(options, args) = parser.parse_args()
if (not options.directory and len(args) < 2) or (options.directory
and len(args) < 1):
parser.print_help()
sys.exit(2)
if options.log:
logg = log_msg.LogMessage(options.log)
else:
logg = log_msg.LogMessage("")
logg.write_starting()
in_files = []
if options.directory:
# create a list of file paths for all files in directory
dirlist = os.listdir(options.directory)
for fname in dirlist:
path = os.path.join(options.directory, fname)
in_files.append(path)
else:
in_files = args[0:-1]
out_file = args[-1]
field_names = [
"id", "unit_id", "insertion_time", "observation_time", "latitude",
"longitude", "elevation", "air_temp", "atmospheric_pressure",
"light_level", "rain_intensity", "relative_humidity", "road_temp"
]
field_types = [
"a32", "a32", "a32", "a32", "f4", "f4", "f4", "f4", "f4", "f4", "f4",
"f4", "f4"
]
field_units = [
None, None, "seconds since 1970-01-01 00 UTC",
"seconds since 1970-01-01 00 UTC", "degrees north", "degrees east",
"meters", "celsius", "hPa", "lux", None, "percent", "celsius"
]
data = []
missing = [
"", "", "", "", -9999., -9999., -9999., -9999., -9999., -9999., -9999.,
-9999., -9999.
]
sizes = [
None, None, None, None, None, None, None, None, None, None, None, None,
None
]
field_object = fields.Fields(data, field_names, field_types, field_units,
missing, sizes)
logg.write_info("parsing first input file %s" % in_files[0])
wxt_parse = wxtParse(in_files[0])
observation_sets = ObservationSets("observation_time", field_object,
wxt_parse.observation_set_dict)
logg.write_info("parsing input files")
for in_file in in_files[1:]:
logg.write_info("parsing %s" % in_file)
wxt_parse = wxtParse(in_file)
new_observation_sets = ObservationSets("observation_time",
field_object,
wxt_parse.observation_set_dict)
observation_sets.merge(new_observation_sets)
if options.cdf:
array = observation_sets.get_field_data()
for date_pairs in Dates:
choose = numpy.logical_and(
numpy.char.greater_equal(array["observation_time"],
date_pairs[0]),
numpy.char.less(array["observation_time"], date_pairs[1]))
array_selection = array[choose]
if array_selection.shape[0] == 0:
continue
data_fields = []
for field_name in field_names:
data_fields.append(array_selection[field_name])
field_object.data = data_fields
date_string = date_pairs[0][0:4] + date_pairs[0][5:7] + date_pairs[
0][8:10] + date_pairs[0][11:13]
out_file_name = "%s.%s.nc" % (out_file, date_string)
logg.write_info("writing netcdf file %s" % out_file_name)
wxt_cdf.WxtCdf(out_file_name, field_object)
else:
fp_out = file(out_file, "w")
observation_sets.write_ascii(fp_out)
fp_out.close()
logg.write_ending()
def parse_hourly(begin_date, end_date, in_files, out_file, cdfout, logg):
files_written = []
# Formulate date list
delta_seconds = 3600
single_dates = tim.date_list_time_delta_iso(begin_date, end_date,
delta_seconds)
if len(single_dates) <= 1:
logg.write_error("date list too small")
return files_written
date_pairs = []
for ind in range(len(single_dates) - 1):
date_pairs.append((single_dates[ind], single_dates[ind + 1]))
field_names = [
"id", "unit_id", "insertion_time", "observation_time", "latitude",
"longitude", "elevation", "air_temp", "atmospheric_pressure",
"light_level", "rain_intensity", "relative_humidity", "road_temp"
]
field_types = [
"a32", "a32", "a32", "a32", "f4", "f4", "f4", "f4", "f4", "f4", "f4",
"f4", "f4"
]
field_units = [
None, None, "seconds since 1970-01-01 00 UTC",
"seconds since 1970-01-01 00 UTC", "degrees north", "degrees east",
"meters", "celsius", "hPa", "lux", None, "percent", "celsius"
]
data = []
missing = [
"", "", "", "", -9999., -9999., -9999., -9999., -9999., -9999., -9999.,
-9999., -9999.
]
sizes = [
None, None, None, None, None, None, None, None, None, None, None, None,
None
]
field_object = fields.Fields(data, field_names, field_types, field_units,
missing, sizes)
logg.write_info("parsing first input file %s" % in_files[0])
wxt_parse = wxtParse(in_files[0])
observation_sets = ObservationSets("observation_time", field_object,
wxt_parse.observation_set_dict)
logg.write_info("parsing input files")
for ind, in_file in enumerate(in_files[1:]):
if ind > 0 and (ind % 1000) == 0:
logg.write_info("finished parsing %d out of %d files" %
(ind, len(in_files)))
wxt_parse = wxtParse(in_file)
new_observation_sets = ObservationSets("observation_time",
field_object,
wxt_parse.observation_set_dict)
observation_sets.merge(new_observation_sets)
if cdfout:
array = observation_sets.get_field_data()
for date_pair in date_pairs:
choose = numpy.logical_and(
numpy.char.greater_equal(array["observation_time"],
date_pair[0]),
numpy.char.less(array["observation_time"], date_pair[1]))
array_selection = array[choose]
if array_selection.shape[0] == 0:
continue
data_fields = []
for field_name in field_names:
data_fields.append(array_selection[field_name])
field_object.data = data_fields
date_string = date_pair[0][0:4] + date_pair[0][5:7] + date_pair[0][
8:10] + date_pair[0][11:13]
out_file_name = "%s.%s.nc" % (out_file, date_string)
logg.write_info("writing netcdf file %s" % out_file_name)
files_written.append(out_file_name)
wxt_cdf.WxtCdf(out_file_name, field_object)
else:
fp_out = file(out_file, "w")
files_written.append(out_file)
observation_sets.write_ascii(fp_out)
fp_out.close()
return files_written