def wait_for_disarm(vehicle, timeout=180):
"""
Wait for the vehicle to disarm and log every 2 sec.
Args:
vehicle(dronekit.Vehicle): the copter to be controlled.
timeout(int): seconds before invoking forced exit. Default 30 seconds.
"""
wait_count = 0
sleep_period = 1
log_period = 30
timeout_limit = timeout / sleep_period
log_count = log_period / sleep_period
while vehicle.armed:
if wait_count % log_count == 0: # 10sec period logging
util.log_info("Waiting for the copter to disarm.")
if wait_count >= timeout_limit:
util.log_warning("Wait timeout! Exit script now!")
break
time.sleep(sleep_period)
wait_count = wait_count + 1
def set_mode(vehicle, mode):
"""
Set the vehicle's flight modes. 200ms period state validation.
Args:
vehicle(dronekit.Vehicle): the vehicle to be controlled.
mode(str): flight mode string, supported by the firmware.
Returns:
bool: True if success, False if failed.
Failure will set shared.status['abort'].
"""
util.log_info("Setting %s." % mode)
shared.status['manual_mode'] = mode
vehicle.mode = VehicleMode(mode)
wait_count = 0
while True:
time.sleep(.2)
wait_count = wait_count + 1
if vehicle.mode.name == mode:
return True
elif wait_count >= 45:
util.log_warning("Unable to set %s. Assume link lost." % mode)
shared.status['abort'] = True
return False
elif wait_count % 15 == 0:
util.log_warning("Retry setting %s" % mode)
vehicle.mode = VehicleMode(mode) # resend command
def __add_module_map(self):
"""Create and add a modulemap to the framework_modules_path"""
log_info('adding modulemap')
os.makedirs(self.framework_modules_path)
module_map_filepath = '{}/module.modulemap'.format(self.framework_modules_path)
open(module_map_filepath, 'w').write(self.get_module_map())
def post_msg():
util.log_info('web_api_server', 'recieve call for post message')
ty = request.form['type']
if ty == 'shutdown':
subprocess.call('sudo', 'halt')
if ty == 'reboot':
subprocess.call('sudo', 'reboot')
if ty == 'setVolumn':
volumn = request.form['volumn']
subprocess.Popen(['amixer', 'set', 'Master', volumn],
stdout=subprocess.PIPE)
if ty == 'setBootTimes':
model = request.form['model']
if model == 'daily':
openTime = request.form['openTime'] + ':00'
closeTime = request.form['closeTime'] + ':00'
if not serial_port.instance.setDailyTIme(openTime, closeTime):
return 'fail'
if model == 'manual':
return_val = serial_port.instance.setModeM()
time.sleep(10)
util.log_info("api return serial port", return_val)
if return_val == False:
return 'fail'
else:
return 'success'
if ty == 'currAppId':
config.current_app_id = request.form['appId']
return 'ok'
def remove_assignments_for_student(account_id):
aids = util.select('Assignment', ('assignment_id'), {'account_id': account_id},
classname=Assignment.__name__)
for aid in aids:
SubmissionFile.remove_files(aid)
nbr_rows_del = util.delete('Assignment',{'account_id':account_id},classname=Assignment.__name__)
util.log_info('Deleted {} assignments where account_id={}'.format(nbr_rows_del,account_id))
def add_info_plist(self):
"""Create and add and Info plist to the framework path"""
log_info('adding Info.plist')
open('{}/Info.plist'.format(self.framework_path), 'w').write(
self.__info_plist)
def __fix_rpaths(self):
"""Change the rpaths of a given dylib.
Change the rpaths to match our framework
encapsulations, essentially keeping the run paths
intact.
"""
log_info('fixing rpaths')
# list the rpaths of this dylib
rpaths = Popen(['otool', '-L', '{}/{}'.format(self.framework_path, self.name)], stdout=PIPE).communicate()[0]
# iterate through each rpath
# (ignore the first two lines, as this
# output is purely informative)
change_library_identification_name(
id='@rpath/{}.framework/{}'.format(self.name, self.name),
library_path=self.bin_path)
for rpath in rpaths.splitlines()[2:]:
if rpath.strip().startswith('@rpath'):
match = re.match(r'.*@rpath\/(.*).dylib', rpath)
if match is not None:
raw_dylib = match.group(1)
real_rpath = re.match(r'.*@rpath\/.*.dylib', rpath.strip()).group(0)
change_library_rpath(
old_rpath=real_rpath,
new_rpath='@rpath/{}.framework/{}'.format(raw_dylib, raw_dylib),
library_path=self.bin_path)
def get_sensor_info():
try:
return Adafruit_DHT.read_retry(Adafruit_DHT.DHT11, SENSOR_PIN)
except Exception as e:
util.log_info('Error while getting dht11 info. Error=[{}]'.format(
str(e)))
return (1, 2)
def dig_info(flight):
try:
if len(flight.find_elements_by_class_name('colAirports')) != 1:
util.log_info(u'AIRCHINA: 非直达航班')
return None
from_airport, to_airport = flight.\
find_elements_by_class_name('colAirports')[0].\
text.strip().split('-')
flight_no = flight.find_element_by_class_name('colFlight').text.\
strip()
from_time = flight.find_element_by_class_name('colDepart').text.\
strip()[0:5]
to_time = flight.find_element_by_class_name('colArrive').text.\
strip()[0:5]
prices = [
int(x.text.strip().replace(',', ''))
for x in flight.find_elements_by_class_name('colPrice')
]
if not prices:
util.log_error(u'AIRCHINA: 抓取价格失败')
return None
price = min(prices)
from_time_pair = [int(x) for x in from_time.split(':')]
to_time_pair = [int(x) for x in to_time.split(':')]
if to_time_pair[0] < from_time_pair[0]:
to_time_pair[0] += 24
return [
AIRLINE_NAME[Website.AIRCHINA], flight_no, from_airport, from_time,
from_time_pair, to_airport, to_time, to_time_pair, price
]
except Exception as e:
util.log_error('AIRCHINA: ' + str(e))
return None
def _wait_home_origin(xbee):
"""
Wait for the HOME_ORIGIN sent by the GCS.
`HOME_ORIGIN` is not the internal `home` set by the flight controller, it is
a high-level reference coordinate, used in the application algorithms.
Note that this is a blocking wait, so the program is stuck indefinately unless
stopped by `EXIT` command or killed by KeyboardInterrupt.
Args:
xbee(xbee.Zigbee): the XBee communication interface.
Returns:
bool: True if success, False if killed by `EXIT` command.
"""
util.log_info("Waiting HOME_ORIGIN.")
wait_count = 0
while not shared.home_origin:
time.sleep(1)
wait_count = wait_count + 1
if shared.status['command'] == 'EXIT':
comm.xbee_broadcast(xbee,
"IFO,%s abort takeoff." % shared.AGENT_ID)
util.log_info("'EXIT' received. Abort takeoff.")
return False
if wait_count >= 10:
wait_count = 0
comm.xbee_broadcast(
xbee, "IFO,%s awaiting HOME_ORIGIN." % shared.AGENT_ID)
return True
def dig_info(flight):
try:
if len(flight.find_elements_by_class_name('colAirports')) != 1:
util.log_info(u'AIRCHINA: 非直达航班')
return None
from_airport, to_airport = flight.\
find_elements_by_class_name('colAirports')[0].\
text.strip().split('-')
flight_no = flight.find_element_by_class_name('colFlight').text.\
strip()
from_time = flight.find_element_by_class_name('colDepart').text.\
strip()[0:5]
to_time = flight.find_element_by_class_name('colArrive').text.\
strip()[0:5]
prices = [int(x.text.strip().replace(',', '')) for x
in flight.find_elements_by_class_name('colPrice')]
if not prices:
util.log_error(u'AIRCHINA: 抓取价格失败')
return None
price = min(prices)
from_time_pair = [int(x) for x in from_time.split(':')]
to_time_pair = [int(x) for x in to_time.split(':')]
if to_time_pair[0] < from_time_pair[0]:
to_time_pair[0] += 24
return [AIRLINE_NAME[Website.AIRCHINA], flight_no, from_airport,
from_time, from_time_pair, to_airport, to_time, to_time_pair,
price]
except Exception as e:
util.log_error('AIRCHINA: ' + str(e))
return None
def set_boot_power(self):
try:
bat = serial_port.instance.getBat()
util.log_info("bat", bat)
if bat is not None:
update_voltage_url = '%s/%s/Device/UpdateVoltage?id=%s' % (
util.util_remote_service(config.const_api_name_resouce),
config.const_api_name_resouce, config.const_service_id)
batValue = bat[3:7]
feedback = requests.post(
update_voltage_url,
data=batValue,
headers={'Content-Type': 'application/json'})
except Exception as err:
util.log_error("get_bat", err)
try:
device_info = self.get_device_info()
config.const_is_set_powerofboot = util.get_cached_version(
config.const_is_set_powerofboot_name)
if config.const_is_set_powerofboot is None and device_info[
'has_power_control'] is not None or config.const_is_set_powerofboot != str(
device_info['has_power_control']):
config.const_is_set_powerofboot = device_info[
'has_power_control']
util.set_cached_version(config.const_is_set_powerofboot_name,
device_info['has_power_control'])
if config.const_is_set_powerofboot == 'True':
is_set_successfully = False
if device_info['power_setting'] == 1:
if device_info['timer']['switch_mode'] == 0:
is_set_successfully = serial_port.instance.setModeM()
if device_info['timer']['switch_mode'] == 1:
start = device_info['timer']['boot_time'].replace(
'T', ' ')
start_yms = time.strftime(
'%H%M%S', time.strptime(start,
'%Y-%m-%d %H:%M:%S'))
end = device_info['timer']['shutdown_time'].replace(
'T', ' ')
end_yms = time.strftime(
'%H%M%S', time.strptime(end, '%Y-%m-%d %H:%M:%S'))
is_set_successfully = serial_port.instance.setDailyTIme(
start_yms, end_yms)
if is_set_successfully:
url = '%s/%s/Device/SetupPowerControl?id=%s' % (
util.util_remote_service(
config.const_api_name_resouce),
config.const_api_name_resouce,
config.const_service_id)
feedback = requests.post(
url,
data='2',
headers={'Content-Type': 'application/json'})
except Exception as err:
util.log_error("set_boot", err)
def RegisterCode():
util.log_info('web_api_server', 'recieve call for register code')
url = '%s/%s/WechatQrcode/CreateDeviceActivationQrcode' % (
util.util_remote_service(
config.const_api_name_wechat), config.const_api_name_wechat)
data = util.get_cached_version('mc')
rsp = requests.post(url, json=data)
result = 'data:image/png;base64,' + rsp.json().replace('"', '')
return result
def contentInfos():
util.log_info('web_api_server', 'recieve call for contentInfos')
applist = []
dr = '%s/buildin/vers' % (config.const_client_root())
for root, dirs, files in os.walk(dr, topdown=False):
for name in files:
if name.startswith('rv_'):
ver_name = name.replace('.ver', '')
applist.append(json.loads(util.get_cached_version(ver_name)))
return jsonify(applist)
def _log_and_broadcast(xbee, info):
"""
Log some information at `INFO` level and broadcast via XBee.
Args:
xbee(xbee.Zigbee): the XBee communication interface.
info(str): the information string.
"""
comm.xbee_broadcast(xbee, info)
util.log_info(info)
def worker(task):
module, period, website = task
util.log_info('deal at %s %s %s' % (str(module), str(period), str(website)))
try:
module.fetch(period)
except Exception as e:
util.log_error(WEBSITE_NAME[website] + ' ' + str(e) + ' ' +
str(period) +
traceback.format_exc().replace('\n', ' | '))
time.sleep(5)
def logreg_cost_col_reduce(mat, label, cost):
mh, mw = mat.shape
vh, vw = label.shape
#assert(vh == 1 and vw == mw or vw == 1 and vh == mw)
if (vh != 1 or vw != mw) and (vw != 1 or vh != mw):
util.log_info('%s ==> %s', mat.shape, label.shape)
assert False
block = (min(mw, MAX_THREAD), 1, 1)
grid = (divup(mw, block[0]), 1)
_logreg_cost_col_reduce_(mat, label, cost, I(mw), np.int32(mat.strides[0] / 4), block=block, grid=grid)
def worker(task):
module, period, website = task
util.log_info('deal at %s %s %s' %
(str(module), str(period), str(website)))
try:
module.fetch(period)
except Exception as e:
util.log_error(WEBSITE_NAME[website] + ' ' + str(e) + ' ' +
str(period) +
traceback.format_exc().replace('\n', ' | '))
time.sleep(5)
def __add_headers(self):
"""Add sanitizied headers to the framework_headers_path"""
log_info('adding headers')
os.makedirs(self.framework_headers_path)
# copy each header and then sanitize it
for header in self.module.headers:
copy2(header, self.framework_headers_path)
self.__sanitize_header('{}/{}'.format(self.framework_headers_path, os.path.basename(header)))
def push_ipc(self, bids_and_asks, exchSym):
msg = {
"exchange": "BITMEX",
"exchSym": exchSym,
"timestamp": time.time(),
"bids": bids_and_asks["bids"],
"asks": bids_and_asks["asks"]
}
if args.print:
util.log_info(msg)
else:
self.pushSocket.send_string(json.dumps(msg))
def init_client():
util.log_info('connecting to adafruit.io. username={}'.format(ADAFRUIT_IO_USERNAME))
client = MQTTClient(ADAFRUIT_IO_USERNAME, ADAFRUIT_IO_KEY)
# Setup the callback functions defined above.
client.on_connect = connected
client.on_disconnect = disconnected
client.on_message = message
client.connect()
client.loop_background()
return client
def __post(self, url, data=None, retry=True):
while (True):
ret = {}
try:
ret = self.send_post(url, data)
except Exception, err:
if retry:
again = True
util.log_info('retry https post:%s because of %s' %
(url, err.message))
else:
raise err
return ret
def qrByUnique(unique):
util.log_info('web_api_server', 'recieve call for qrByUnique')
ret_obj = {
'Id':
str(uuid.uuid1()),
'FileName':
unique,
'FileUrl':
'%s/Content/QrCodeResources/%s' % (config.const_client_root(), unique),
'HostUrl':
'%s/Content/QrCodeResources/%s' %
(config.const_client_web_server_root, unique)
}
return json.dumps(ret_obj)
def getCurrentPowerSettingModel():
util.log_info('web_api_server', 'recieve current mdoel of power setting')
model = serial_port.instance.getMode()
if model == "MODELD":
openTime = serial_port.instance.getDailyOpenTime()
closeTime = serial_port.instance.getDailyCloseTIme()
ret_obj = {
model: "MODELD",
"opemTime": openTime,
"closeTime": closeTime,
}
return json.dumps(ret_obj)
ret_obj = {model: "MODEM"}
return json.dumps(ret_obj)
def processCmd(self,cmd):
try:
if self.serialport is not None:
if self.serialport.is_open == False:
self.serialport.open()
cmd += '\r'
self.serialport.write(cmd.encode())
result = self.serialport.readline()
util.log_info("fck_result", result.decode())
return result
return None
except Exception as err:
util.log_error("fck", err)
return None
def rx_data_handler(name, packet):
"""
Callback function for incoming XBee transmission.
Args:
name(str): name to identify the callback function in a dispatcher.
packet(dict): package received from the XBee API.
"""
# Split datapack header and payload -- Small misc packs.
if packet['rf_data'][0:3] == 'CTR': # control command pack
recv_pack = packet['rf_data'].split(',')
shared.status['command'] = recv_pack[1]
util.log_info("CMD: %s" % recv_pack[1])
elif packet['rf_data'][0:3] == 'IFO': # information string pack
recv_pack = packet['rf_data'].split(',')
util.log_info("IFO: %s" % recv_pack[1])
elif packet['rf_data'][0:3] == 'PRM': # parameter pack
recv_pack = packet['rf_data'].split(',')
if recv_pack[1][0:3] == 'MPC': # currently only DMPC is available
shared.param_mpc = mas.ParamMPC(unpack('=3s3f2i', recv_pack[1]))
util.log_info("New MPC param received: %s" % shared.param_mpc)
shared.status['new_param'] = True
elif packet[
'rf_data'] == 'CLR_RDV': # clear rendezvous command, no comma split.
shared.rendezvous = None
util.log_info("'CLR_RDV' received, clear rendezvous")
# guarantee packet integrity (payload length) -- Actual agent data pack
elif len(packet['rf_data']) == shared.PAYLOAD_LENGTH:
#from binascii import hexlify # logging binary bytes
#util.log_debug("DATAPACK: %s" % hexlify(packet['rf_data']))
# unpack data into lists and wrap into local datapacks
recv_pack = WrappedData(unpack('=3s2d5f', packet['rf_data']))
util.log_debug("%s" % recv_pack)
if recv_pack.ID == 'TAR': # target point pack
shared.rendezvous = recv_pack
util.log_debug("Rendezvous coordinate received.")
elif recv_pack.ID == 'ORG': # HOME_ORIGIN pack
shared.home_origin = recv_pack.location_global_relative
util.log_info("HOME_ORIGIN set: %s." % shared.home_origin)
elif recv_pack.ID[0] == 'A': # normal neighbor begins with 'Axx'
shared.neighbors[recv_pack.ID] = recv_pack
def run(self):
"""
Run the thread and purge periodically.
"""
util.log_info("Purge Started.")
count = 0
while not self._stopflag:
time.sleep(.2)
count = count + 1
count = count % 15 # Clear neighbors table each 3s
if count == 0:
self.table.clear()
if shared.status['thread_flag'] & shared.PURGE_FLAG:
util.log_info("Purge killed by flag.")
self._stopflag = True
def create(self, platform, variant, output_dir):
super(FrameworkModule, self).create(platform, variant, output_dir)
self.is_created = False
log_info('creating framework module: {}'.format(self.module.name))
self.framework_headers_path = '{}/Headers'.format(self.framework_path)
self.framework_modules_path = '{}/Modules'.format(self.framework_path)
# add headers to the appropriate directory
self.__add_headers()
# create and add module map to the appropriate directory
self.__add_module_map()
# create and add info plist to the appropriate directory
self.add_info_plist()
self.is_created = True
return self
def run(self):
"""
Run the subscriber and listen to the publisher every 5ms.
"""
util.log_info("Subscriber started for %s." % self.agent_id)
while not self._stopflag:
time.sleep(.005)
try:
[tag, payload] = self.sub.recv_multipart(flags=1)
if tag == 'XBEE':
# package to be consistent with <rx_data_handler>
packet = {'rf_data': payload}
rx_data_handler('sub_rx', packet)
except Again: # zmq.error Exception
pass
if shared.status['thread_flag'] & shared.SUBSCRIBER_FLAG:
self._stopflag = True
util.log_info("Subscriber killed by flag.")
def qrInfos():
util.log_info('web_api_server', 'recieve call for qrInfos')
ret = []
for root, dirs, files in os.walk('%s/content/QrCodeResources/'):
for name in files:
item = {
'Id':
str(uuid.uuid1()),
'FileName':
name,
'FileUrl':
'%s/Content/QrCodeResources/%s' %
(config.const_client_root(), name),
'HostUrl':
'%s/Content/QrCodeResources/%s' %
(config.const_client_web_server_root, name)
}
ret.append(item)
return json.dumps(ret)
def run(self):
"""
Broadcast the data at 16Hz.
Pack all the data into a binary pack using `struct.pack`. The payload will
have a constant 39 byte length. Coordinates should be Ctype-double to
guarantee precision. Altitude and velocity should work okay with float.
The data list:
(ID is also the Key in <neighbors>)
----------------------------------
item: Python-type: C-type length(B) offset:
agent_id string char[] 3 0
lat float double 8 1
lon float double 8 2
msl_alt float float 4 3
rel_alt float float 4 4
vx float float 4 5
vy float float 4 6
vz float float 4 7
-----------------------------------
"""
util.log_info("Broadcast Started.")
while not self._stopflag:
time.sleep(.0625) # 16Hz
if self._vehicle.armed: # Only broadcast while vehicle is armed
package = pack(
'=3s2d5f', self._agent_id,
self._vehicle.location.global_frame.lat,
self._vehicle.location.global_frame.lon,
self._vehicle.location.global_frame.alt,
self._vehicle.location.global_relative_frame.alt,
self._vehicle.velocity[0], self._vehicle.velocity[1],
self._vehicle.velocity[2])
xbee_broadcast(self._xbee, package)
if shared.status['thread_flag'] & shared.BROADCAST_FLAG:
self._stopflag = True
util.log_info("Broadcast killed by flag.")
def process_cghub(config,
type_uri='detail',
log=None,
removedups=False,
limit=-1,
verbose=False,
print_response=False):
"""
return type:
tumor_type2cghub_records: organizes the cghub record classes per tumor type
"""
log_info(log, 'begin process cghub')
module = import_module(config['cghub_module'])
mappings = config['metadata_locations']['cghub']
cghub_records, _ = module.main(mappings['study'],
log=log,
removedups=removedups,
limit=limit)
tumor_type2cghub_records = {}
count = 0
seen_bad_codes = set()
for cghub_record in cghub_records:
if 0 == count % 8192:
log_info(log, '\tprocess %s cghub records' % (count))
count += 1
tumor_type2cghub_records.setdefault(
cghub_record.disease_abbr, []).append(
create_cghub_metadata(mappings, cghub_record, seen_bad_codes,
log))
log_info(log, 'finished process cghub: %s total records' % (count))
return tumor_type2cghub_records
def create(self, platform, variant, output_dir):
if self.is_created:
log_error('framework {} has already been created'.format(self.name))
return self
log_info('creating framework: {}'.format(self.name))
self.framework_path = '{}/{}/{}/{}.framework'.format(
output_dir, platform.full_name, variant.name, self.name)
self.bin_path = '{}/{}'.format(self.framework_path, self.name)
# set the framework path and make our dylib.framework
# directory
os.makedirs(self.framework_path)
self.abs_path = os.path.abspath(self.framework_path)
# copy the binary to the new path
copy2(self.library_path, self.bin_path)
# fix run paths, since we will be
# encapsulating each dylib
if self.should_fix_rpaths:
self.__fix_rpaths()
self.is_created = True
return self
def process_cghub(config, type_uri = 'detail', log = None, removedups = False, limit = -1, verbose = False, print_response = False):
"""
return type:
tumor_type2cghub_records: organizes the cghub record classes per tumor type
"""
log_info(log, 'begin process cghub')
module = import_module(config['cghub_module'])
mappings = config['metadata_locations']['cghub']
cghub_records, _ = module.main(mappings['study'], log = log, removedups = removedups, limit = limit)
tumor_type2cghub_records = {}
count = 0
seen_bad_codes = set()
for cghub_record in cghub_records:
if 0 == count % 8192:
log_info(log, '\tprocess %s cghub records' % (count))
count += 1
tumor_type2cghub_records.setdefault(cghub_record.disease_abbr, []).append(create_cghub_metadata(mappings, cghub_record, seen_bad_codes, log))
log_info(log, 'finished process cghub: %s total records' % (count))
return tumor_type2cghub_records
def statistics(log, filename2cghubRecords, minmaxsize, verbose):
states = {}
centers = {}
studies = {}
sampleIDs = [{}, {}, {}, {}, {}, {}, {}, {}]
diseases = {}
analyte_codes = {}
sample_types = {}
strategies = {}
platforms = {}
refassems = {}
models = {}
for record in filename2cghubRecords.itervalues():
states[record.state] = states.setdefault(record.state, 0) + 1
centers[record.center_name] = centers.setdefault(record.center_name, 0) + 1
studies[record.study] = studies.setdefault(record.study, 0) + 1
diseases[record.disease_abbr] = diseases.setdefault(record.disease_abbr, 0) + 1
analyte_codes[record.analyte_code] = analyte_codes.setdefault(record.analyte_code, 0) + 1
sample_types[record.sample_type] = sample_types.setdefault(record.sample_type, 0) + 1
strategies[record.library_strategy] = strategies.setdefault(record.library_strategy, 0) + 1
platforms[record.platform] = platforms.setdefault(record.platform, 0) + 1
refassems[record.refassem_short_name] = refassems.setdefault(record.refassem_short_name, 0) + 1
models[record.platform_full_name] = models.setdefault(record.platform_full_name, 0) + 1
try:
fields = record.legacy_sample_id.split('-')
for index, field in enumerate(fields[:-3]):
sampleIDs[index][field] = sampleIDs[index].setdefault(field, 0) + 1
for index, field in enumerate(fields[-3:]):
sampleIDs[index + len(fields[:-3])][field] = sampleIDs[index + len(fields[:-3])].setdefault(field, 0) + 1
except:
util.log_exception(log, 'problem splitting %s(%s:%s)' % (record.legacy_sample_id, index, field))
util.log_info(log, '\nStates')
count = 0
for state, value in states.iteritems():
if count < 15:
count += 1
util.log_info(log, '%s: %s' % (state, value))
else:
util.log_info(log, '\t(of %s)' % (len(states.keys())))
break
util.log_info(log, '')
util.log_info(log, 'Centers')
count = 0
for center, value in centers.iteritems():
if count < 15:
count += 1
util.log_info(log, '%s: %s' % (center, value))
else:
util.log_info(log, ' (of %s)' % (len(centers.keys())))
break
util.log_info(log, '')
util.log_info(log, 'Studies')
count = 0
for studie, value in studies.iteritems():
if count < 15:
count += 1
util.log_info(log, '%s: %s' % (studie, value))
else:
util.log_info(log, '(of %s)' % (len(studies.keys())))
break
util.log_info(log, '')
if verbose:
util.log_info(log, 'Sample ids:')
count = 0
for sampleMap in sampleIDs:
util.log_info(log, 'next part:')
for sampleID, value in sampleMap.iteritems():
if count < 15:
count += 1
util.log_info(log, '%s: %s' % (sampleID, value))
else:
util.log_info(log, '(of %s)' % (len(sampleMap.keys())))
break
util.log_info(log, '')
count = 0
util.log_info(log, 'Diseases:')
count = 0
for disease, value in diseases.iteritems():
count += 1
util.log_info(log, '%s: %s' % (disease, value))
util.log_info(log, '')
util.log_info(log, 'Analyte codes:')
count = 0
for analyte_code, value in analyte_codes.iteritems():
if count < 15:
count += 1
util.log_info(log, '%s: %s' % (analyte_code, value))
else:
util.log_info(log, '(of %s)' % (len(analyte_codes.keys())))
break
util.log_info(log, '')
util.log_info(log, 'Sample types')
count = 0
for sample_type, value in sample_types.iteritems():
if count < 15:
count += 1
util.log_info(log, '%s: %s' % (sample_type, value))
else:
util.log_info(log, '(of %s)' % (len(sample_types.keys())))
break
util.log_info(log, '')
util.log_info(log, 'Strategies:')
count = 0
for strategie, value in strategies.iteritems():
if count < 15:
count += 1
util.log_info(log, '%s: %s' % (strategie, value))
else:
util.log_info(log, '(of %s)' % (len(strategies.keys())))
break
util.log_info(log, '')
util.log_info(log, 'Platforms:')
count = 0
for platform, value in platforms.iteritems():
if count < 15:
count += 1
util.log_info(log, '%s: %s' % (platform, value))
else:
util.log_info(log, '(of %s)' % (len(platforms.keys())))
break
util.log_info(log, '')
util.log_info(log, 'Reference Assembles:')
count = 0
for refassem, value in refassems.iteritems():
if count < 15:
count += 1
util.log_info(log, '%s: %s' % (refassem, value))
else:
util.log_info(log, '(of %s)' % (len(refassems.keys())))
break
util.log_info(log, '')
util.log_info(log, 'Models:')
count = 0
for model, value in models.iteritems():
if count < 15:
count += 1
util.log_info(log, '%s: %s' % (model, value))
else:
util.log_info(log, '(of %s)' % (len(models.keys())))
break
util.log_info(log, '')
util.log_info(log, '\n\t\tmax: %s\n\t\tmin: %s' % (minmaxsize['max'].write(), minmaxsize['min'].write()))
def main(platform, type_uri = 'detail', log = None, removedups = False, limit = -1, verbose = False, print_response = False):
util.log_info(log, 'begin reading cghub archive')
filename2cghubRecords = {}
minmaxsize = {'min': CGHubFileInfo('', 500000000000, ''), 'max': CGHubFileInfo('', 1, '')}
try:
# archives = util.getURLData(manifest_uri, 'latest_manifest', log)
response = urllib.urlopen(manifest_uri)
archives = response.read()
lines = archives.split('\n')
util.log_info(log, '\tarchive size is %s with %s lines' % (len(archives), len(lines)))
util.log_info(log, '\n\t' + '\n\t'.join(lines[:10]))
except Exception as e:
util.log_exception(log, 'problem fetching latest_manifest: %s')
raise e
headers = lines[0].split('\t')
column_index2header = {}
for index, header in enumerate(headers):
if header in header2record_index.keys():
column_index2header[index] = header
count = 0
dupcount = 0
for line in lines[1:]:
if not line:
continue
if 0 == count % 4096:
util.log_info(log, 'processed %s records' % (count))
count += 1
fields = line.split('\t')
header2record = {}
try:
for index in column_index2header.keys():
header2record[header2record_index[column_index2header[index]]] = fields[index]
except Exception as e:
util.log_info(log, 'problem with parsing line(%s): %s' % (count, line))
if platform not in header2record[CGHubRecordInfo.study_index]:
continue
header2record.update(index2none)
record = CGHubRecordInfo(header2record)
filename = header2record[CGHubRecordInfo.bamFilename_index]
if removedups and filename in filename2cghubRecords:
if 'Live' == header2record[CGHubRecordInfo.state_index]:
dupcount += 1
# check the dates and keep the latest
currentdate = createDateTime(filename2cghubRecords[filename].upload_date)
newdate = createDateTime(record.upload_date)
if currentdate < newdate:
filename2cghubRecords[filename] = record
else:
filename2cghubRecords[filename] = record
if 'Live' == record.state:
if minmaxsize['min'].filesize > record.files['bam'].filesize and record.files['bam'].filesize:
minmaxsize['min'] = record.files['bam']
if minmaxsize['max'].filesize < record.files['bam'].filesize:
minmaxsize['max'] = record.files['bam']
if not record.files['bam'].filesize:
util.log_info(log, 'no file size: %s--%s' % (record.write(), record.files['bam'].write()))
statistics(log, filename2cghubRecords, minmaxsize, verbose)
util.log_info(log, 'finished reading cghub archive. %s total records, %s duplicates' % (count, dupcount))
return filename2cghubRecords.values(), minmaxsize
