def wait_for_disarm(vehicle, timeout=180): """ Wait for the vehicle to disarm and log every 2 sec. Args: vehicle(dronekit.Vehicle): the copter to be controlled. timeout(int): seconds before invoking forced exit. Default 30 seconds. """ wait_count = 0 sleep_period = 1 log_period = 30 timeout_limit = timeout / sleep_period log_count = log_period / sleep_period while vehicle.armed: if wait_count % log_count == 0: # 10sec period logging util.log_info("Waiting for the copter to disarm.") if wait_count >= timeout_limit: util.log_warning("Wait timeout! Exit script now!") break time.sleep(sleep_period) wait_count = wait_count + 1
def set_mode(vehicle, mode): """ Set the vehicle's flight modes. 200ms period state validation. Args: vehicle(dronekit.Vehicle): the vehicle to be controlled. mode(str): flight mode string, supported by the firmware. Returns: bool: True if success, False if failed. Failure will set shared.status['abort']. """ util.log_info("Setting %s." % mode) shared.status['manual_mode'] = mode vehicle.mode = VehicleMode(mode) wait_count = 0 while True: time.sleep(.2) wait_count = wait_count + 1 if vehicle.mode.name == mode: return True elif wait_count >= 45: util.log_warning("Unable to set %s. Assume link lost." % mode) shared.status['abort'] = True return False elif wait_count % 15 == 0: util.log_warning("Retry setting %s" % mode) vehicle.mode = VehicleMode(mode) # resend command
def __add_module_map(self): """Create and add a modulemap to the framework_modules_path""" log_info('adding modulemap') os.makedirs(self.framework_modules_path) module_map_filepath = '{}/module.modulemap'.format(self.framework_modules_path) open(module_map_filepath, 'w').write(self.get_module_map())
def post_msg(): util.log_info('web_api_server', 'recieve call for post message') ty = request.form['type'] if ty == 'shutdown': subprocess.call('sudo', 'halt') if ty == 'reboot': subprocess.call('sudo', 'reboot') if ty == 'setVolumn': volumn = request.form['volumn'] subprocess.Popen(['amixer', 'set', 'Master', volumn], stdout=subprocess.PIPE) if ty == 'setBootTimes': model = request.form['model'] if model == 'daily': openTime = request.form['openTime'] + ':00' closeTime = request.form['closeTime'] + ':00' if not serial_port.instance.setDailyTIme(openTime, closeTime): return 'fail' if model == 'manual': return_val = serial_port.instance.setModeM() time.sleep(10) util.log_info("api return serial port", return_val) if return_val == False: return 'fail' else: return 'success' if ty == 'currAppId': config.current_app_id = request.form['appId'] return 'ok'
def remove_assignments_for_student(account_id): aids = util.select('Assignment', ('assignment_id'), {'account_id': account_id}, classname=Assignment.__name__) for aid in aids: SubmissionFile.remove_files(aid) nbr_rows_del = util.delete('Assignment',{'account_id':account_id},classname=Assignment.__name__) util.log_info('Deleted {} assignments where account_id={}'.format(nbr_rows_del,account_id))
def add_info_plist(self): """Create and add and Info plist to the framework path""" log_info('adding Info.plist') open('{}/Info.plist'.format(self.framework_path), 'w').write( self.__info_plist)
def __fix_rpaths(self): """Change the rpaths of a given dylib. Change the rpaths to match our framework encapsulations, essentially keeping the run paths intact. """ log_info('fixing rpaths') # list the rpaths of this dylib rpaths = Popen(['otool', '-L', '{}/{}'.format(self.framework_path, self.name)], stdout=PIPE).communicate()[0] # iterate through each rpath # (ignore the first two lines, as this # output is purely informative) change_library_identification_name( id='@rpath/{}.framework/{}'.format(self.name, self.name), library_path=self.bin_path) for rpath in rpaths.splitlines()[2:]: if rpath.strip().startswith('@rpath'): match = re.match(r'.*@rpath\/(.*).dylib', rpath) if match is not None: raw_dylib = match.group(1) real_rpath = re.match(r'.*@rpath\/.*.dylib', rpath.strip()).group(0) change_library_rpath( old_rpath=real_rpath, new_rpath='@rpath/{}.framework/{}'.format(raw_dylib, raw_dylib), library_path=self.bin_path)
def get_sensor_info(): try: return Adafruit_DHT.read_retry(Adafruit_DHT.DHT11, SENSOR_PIN) except Exception as e: util.log_info('Error while getting dht11 info. Error=[{}]'.format( str(e))) return (1, 2)
def dig_info(flight): try: if len(flight.find_elements_by_class_name('colAirports')) != 1: util.log_info(u'AIRCHINA: 非直达航班') return None from_airport, to_airport = flight.\ find_elements_by_class_name('colAirports')[0].\ text.strip().split('-') flight_no = flight.find_element_by_class_name('colFlight').text.\ strip() from_time = flight.find_element_by_class_name('colDepart').text.\ strip()[0:5] to_time = flight.find_element_by_class_name('colArrive').text.\ strip()[0:5] prices = [ int(x.text.strip().replace(',', '')) for x in flight.find_elements_by_class_name('colPrice') ] if not prices: util.log_error(u'AIRCHINA: 抓取价格失败') return None price = min(prices) from_time_pair = [int(x) for x in from_time.split(':')] to_time_pair = [int(x) for x in to_time.split(':')] if to_time_pair[0] < from_time_pair[0]: to_time_pair[0] += 24 return [ AIRLINE_NAME[Website.AIRCHINA], flight_no, from_airport, from_time, from_time_pair, to_airport, to_time, to_time_pair, price ] except Exception as e: util.log_error('AIRCHINA: ' + str(e)) return None
def _wait_home_origin(xbee): """ Wait for the HOME_ORIGIN sent by the GCS. `HOME_ORIGIN` is not the internal `home` set by the flight controller, it is a high-level reference coordinate, used in the application algorithms. Note that this is a blocking wait, so the program is stuck indefinately unless stopped by `EXIT` command or killed by KeyboardInterrupt. Args: xbee(xbee.Zigbee): the XBee communication interface. Returns: bool: True if success, False if killed by `EXIT` command. """ util.log_info("Waiting HOME_ORIGIN.") wait_count = 0 while not shared.home_origin: time.sleep(1) wait_count = wait_count + 1 if shared.status['command'] == 'EXIT': comm.xbee_broadcast(xbee, "IFO,%s abort takeoff." % shared.AGENT_ID) util.log_info("'EXIT' received. Abort takeoff.") return False if wait_count >= 10: wait_count = 0 comm.xbee_broadcast( xbee, "IFO,%s awaiting HOME_ORIGIN." % shared.AGENT_ID) return True
def dig_info(flight): try: if len(flight.find_elements_by_class_name('colAirports')) != 1: util.log_info(u'AIRCHINA: 非直达航班') return None from_airport, to_airport = flight.\ find_elements_by_class_name('colAirports')[0].\ text.strip().split('-') flight_no = flight.find_element_by_class_name('colFlight').text.\ strip() from_time = flight.find_element_by_class_name('colDepart').text.\ strip()[0:5] to_time = flight.find_element_by_class_name('colArrive').text.\ strip()[0:5] prices = [int(x.text.strip().replace(',', '')) for x in flight.find_elements_by_class_name('colPrice')] if not prices: util.log_error(u'AIRCHINA: 抓取价格失败') return None price = min(prices) from_time_pair = [int(x) for x in from_time.split(':')] to_time_pair = [int(x) for x in to_time.split(':')] if to_time_pair[0] < from_time_pair[0]: to_time_pair[0] += 24 return [AIRLINE_NAME[Website.AIRCHINA], flight_no, from_airport, from_time, from_time_pair, to_airport, to_time, to_time_pair, price] except Exception as e: util.log_error('AIRCHINA: ' + str(e)) return None
def set_boot_power(self): try: bat = serial_port.instance.getBat() util.log_info("bat", bat) if bat is not None: update_voltage_url = '%s/%s/Device/UpdateVoltage?id=%s' % ( util.util_remote_service(config.const_api_name_resouce), config.const_api_name_resouce, config.const_service_id) batValue = bat[3:7] feedback = requests.post( update_voltage_url, data=batValue, headers={'Content-Type': 'application/json'}) except Exception as err: util.log_error("get_bat", err) try: device_info = self.get_device_info() config.const_is_set_powerofboot = util.get_cached_version( config.const_is_set_powerofboot_name) if config.const_is_set_powerofboot is None and device_info[ 'has_power_control'] is not None or config.const_is_set_powerofboot != str( device_info['has_power_control']): config.const_is_set_powerofboot = device_info[ 'has_power_control'] util.set_cached_version(config.const_is_set_powerofboot_name, device_info['has_power_control']) if config.const_is_set_powerofboot == 'True': is_set_successfully = False if device_info['power_setting'] == 1: if device_info['timer']['switch_mode'] == 0: is_set_successfully = serial_port.instance.setModeM() if device_info['timer']['switch_mode'] == 1: start = device_info['timer']['boot_time'].replace( 'T', ' ') start_yms = time.strftime( '%H%M%S', time.strptime(start, '%Y-%m-%d %H:%M:%S')) end = device_info['timer']['shutdown_time'].replace( 'T', ' ') end_yms = time.strftime( '%H%M%S', time.strptime(end, '%Y-%m-%d %H:%M:%S')) is_set_successfully = serial_port.instance.setDailyTIme( start_yms, end_yms) if is_set_successfully: url = '%s/%s/Device/SetupPowerControl?id=%s' % ( util.util_remote_service( config.const_api_name_resouce), config.const_api_name_resouce, config.const_service_id) feedback = requests.post( url, data='2', headers={'Content-Type': 'application/json'}) except Exception as err: util.log_error("set_boot", err)
def RegisterCode(): util.log_info('web_api_server', 'recieve call for register code') url = '%s/%s/WechatQrcode/CreateDeviceActivationQrcode' % ( util.util_remote_service( config.const_api_name_wechat), config.const_api_name_wechat) data = util.get_cached_version('mc') rsp = requests.post(url, json=data) result = 'data:image/png;base64,' + rsp.json().replace('"', '') return result
def contentInfos(): util.log_info('web_api_server', 'recieve call for contentInfos') applist = [] dr = '%s/buildin/vers' % (config.const_client_root()) for root, dirs, files in os.walk(dr, topdown=False): for name in files: if name.startswith('rv_'): ver_name = name.replace('.ver', '') applist.append(json.loads(util.get_cached_version(ver_name))) return jsonify(applist)
def _log_and_broadcast(xbee, info): """ Log some information at `INFO` level and broadcast via XBee. Args: xbee(xbee.Zigbee): the XBee communication interface. info(str): the information string. """ comm.xbee_broadcast(xbee, info) util.log_info(info)
def worker(task): module, period, website = task util.log_info('deal at %s %s %s' % (str(module), str(period), str(website))) try: module.fetch(period) except Exception as e: util.log_error(WEBSITE_NAME[website] + ' ' + str(e) + ' ' + str(period) + traceback.format_exc().replace('\n', ' | ')) time.sleep(5)
def logreg_cost_col_reduce(mat, label, cost): mh, mw = mat.shape vh, vw = label.shape #assert(vh == 1 and vw == mw or vw == 1 and vh == mw) if (vh != 1 or vw != mw) and (vw != 1 or vh != mw): util.log_info('%s ==> %s', mat.shape, label.shape) assert False block = (min(mw, MAX_THREAD), 1, 1) grid = (divup(mw, block[0]), 1) _logreg_cost_col_reduce_(mat, label, cost, I(mw), np.int32(mat.strides[0] / 4), block=block, grid=grid)
def __add_headers(self): """Add sanitizied headers to the framework_headers_path""" log_info('adding headers') os.makedirs(self.framework_headers_path) # copy each header and then sanitize it for header in self.module.headers: copy2(header, self.framework_headers_path) self.__sanitize_header('{}/{}'.format(self.framework_headers_path, os.path.basename(header)))
def push_ipc(self, bids_and_asks, exchSym): msg = { "exchange": "BITMEX", "exchSym": exchSym, "timestamp": time.time(), "bids": bids_and_asks["bids"], "asks": bids_and_asks["asks"] } if args.print: util.log_info(msg) else: self.pushSocket.send_string(json.dumps(msg))
def init_client(): util.log_info('connecting to adafruit.io. username={}'.format(ADAFRUIT_IO_USERNAME)) client = MQTTClient(ADAFRUIT_IO_USERNAME, ADAFRUIT_IO_KEY) # Setup the callback functions defined above. client.on_connect = connected client.on_disconnect = disconnected client.on_message = message client.connect() client.loop_background() return client
def __post(self, url, data=None, retry=True): while (True): ret = {} try: ret = self.send_post(url, data) except Exception, err: if retry: again = True util.log_info('retry https post:%s because of %s' % (url, err.message)) else: raise err return ret
def qrByUnique(unique): util.log_info('web_api_server', 'recieve call for qrByUnique') ret_obj = { 'Id': str(uuid.uuid1()), 'FileName': unique, 'FileUrl': '%s/Content/QrCodeResources/%s' % (config.const_client_root(), unique), 'HostUrl': '%s/Content/QrCodeResources/%s' % (config.const_client_web_server_root, unique) } return json.dumps(ret_obj)
def getCurrentPowerSettingModel(): util.log_info('web_api_server', 'recieve current mdoel of power setting') model = serial_port.instance.getMode() if model == "MODELD": openTime = serial_port.instance.getDailyOpenTime() closeTime = serial_port.instance.getDailyCloseTIme() ret_obj = { model: "MODELD", "opemTime": openTime, "closeTime": closeTime, } return json.dumps(ret_obj) ret_obj = {model: "MODEM"} return json.dumps(ret_obj)
def processCmd(self,cmd): try: if self.serialport is not None: if self.serialport.is_open == False: self.serialport.open() cmd += '\r' self.serialport.write(cmd.encode()) result = self.serialport.readline() util.log_info("fck_result", result.decode()) return result return None except Exception as err: util.log_error("fck", err) return None
def rx_data_handler(name, packet): """ Callback function for incoming XBee transmission. Args: name(str): name to identify the callback function in a dispatcher. packet(dict): package received from the XBee API. """ # Split datapack header and payload -- Small misc packs. if packet['rf_data'][0:3] == 'CTR': # control command pack recv_pack = packet['rf_data'].split(',') shared.status['command'] = recv_pack[1] util.log_info("CMD: %s" % recv_pack[1]) elif packet['rf_data'][0:3] == 'IFO': # information string pack recv_pack = packet['rf_data'].split(',') util.log_info("IFO: %s" % recv_pack[1]) elif packet['rf_data'][0:3] == 'PRM': # parameter pack recv_pack = packet['rf_data'].split(',') if recv_pack[1][0:3] == 'MPC': # currently only DMPC is available shared.param_mpc = mas.ParamMPC(unpack('=3s3f2i', recv_pack[1])) util.log_info("New MPC param received: %s" % shared.param_mpc) shared.status['new_param'] = True elif packet[ 'rf_data'] == 'CLR_RDV': # clear rendezvous command, no comma split. shared.rendezvous = None util.log_info("'CLR_RDV' received, clear rendezvous") # guarantee packet integrity (payload length) -- Actual agent data pack elif len(packet['rf_data']) == shared.PAYLOAD_LENGTH: #from binascii import hexlify # logging binary bytes #util.log_debug("DATAPACK: %s" % hexlify(packet['rf_data'])) # unpack data into lists and wrap into local datapacks recv_pack = WrappedData(unpack('=3s2d5f', packet['rf_data'])) util.log_debug("%s" % recv_pack) if recv_pack.ID == 'TAR': # target point pack shared.rendezvous = recv_pack util.log_debug("Rendezvous coordinate received.") elif recv_pack.ID == 'ORG': # HOME_ORIGIN pack shared.home_origin = recv_pack.location_global_relative util.log_info("HOME_ORIGIN set: %s." % shared.home_origin) elif recv_pack.ID[0] == 'A': # normal neighbor begins with 'Axx' shared.neighbors[recv_pack.ID] = recv_pack
def run(self): """ Run the thread and purge periodically. """ util.log_info("Purge Started.") count = 0 while not self._stopflag: time.sleep(.2) count = count + 1 count = count % 15 # Clear neighbors table each 3s if count == 0: self.table.clear() if shared.status['thread_flag'] & shared.PURGE_FLAG: util.log_info("Purge killed by flag.") self._stopflag = True
def create(self, platform, variant, output_dir): super(FrameworkModule, self).create(platform, variant, output_dir) self.is_created = False log_info('creating framework module: {}'.format(self.module.name)) self.framework_headers_path = '{}/Headers'.format(self.framework_path) self.framework_modules_path = '{}/Modules'.format(self.framework_path) # add headers to the appropriate directory self.__add_headers() # create and add module map to the appropriate directory self.__add_module_map() # create and add info plist to the appropriate directory self.add_info_plist() self.is_created = True return self
def run(self): """ Run the subscriber and listen to the publisher every 5ms. """ util.log_info("Subscriber started for %s." % self.agent_id) while not self._stopflag: time.sleep(.005) try: [tag, payload] = self.sub.recv_multipart(flags=1) if tag == 'XBEE': # package to be consistent with <rx_data_handler> packet = {'rf_data': payload} rx_data_handler('sub_rx', packet) except Again: # zmq.error Exception pass if shared.status['thread_flag'] & shared.SUBSCRIBER_FLAG: self._stopflag = True util.log_info("Subscriber killed by flag.")
def qrInfos(): util.log_info('web_api_server', 'recieve call for qrInfos') ret = [] for root, dirs, files in os.walk('%s/content/QrCodeResources/'): for name in files: item = { 'Id': str(uuid.uuid1()), 'FileName': name, 'FileUrl': '%s/Content/QrCodeResources/%s' % (config.const_client_root(), name), 'HostUrl': '%s/Content/QrCodeResources/%s' % (config.const_client_web_server_root, name) } ret.append(item) return json.dumps(ret)
def run(self): """ Broadcast the data at 16Hz. Pack all the data into a binary pack using `struct.pack`. The payload will have a constant 39 byte length. Coordinates should be Ctype-double to guarantee precision. Altitude and velocity should work okay with float. The data list: (ID is also the Key in <neighbors>) ---------------------------------- item: Python-type: C-type length(B) offset: agent_id string char[] 3 0 lat float double 8 1 lon float double 8 2 msl_alt float float 4 3 rel_alt float float 4 4 vx float float 4 5 vy float float 4 6 vz float float 4 7 ----------------------------------- """ util.log_info("Broadcast Started.") while not self._stopflag: time.sleep(.0625) # 16Hz if self._vehicle.armed: # Only broadcast while vehicle is armed package = pack( '=3s2d5f', self._agent_id, self._vehicle.location.global_frame.lat, self._vehicle.location.global_frame.lon, self._vehicle.location.global_frame.alt, self._vehicle.location.global_relative_frame.alt, self._vehicle.velocity[0], self._vehicle.velocity[1], self._vehicle.velocity[2]) xbee_broadcast(self._xbee, package) if shared.status['thread_flag'] & shared.BROADCAST_FLAG: self._stopflag = True util.log_info("Broadcast killed by flag.")
def process_cghub(config, type_uri='detail', log=None, removedups=False, limit=-1, verbose=False, print_response=False): """ return type: tumor_type2cghub_records: organizes the cghub record classes per tumor type """ log_info(log, 'begin process cghub') module = import_module(config['cghub_module']) mappings = config['metadata_locations']['cghub'] cghub_records, _ = module.main(mappings['study'], log=log, removedups=removedups, limit=limit) tumor_type2cghub_records = {} count = 0 seen_bad_codes = set() for cghub_record in cghub_records: if 0 == count % 8192: log_info(log, '\tprocess %s cghub records' % (count)) count += 1 tumor_type2cghub_records.setdefault( cghub_record.disease_abbr, []).append( create_cghub_metadata(mappings, cghub_record, seen_bad_codes, log)) log_info(log, 'finished process cghub: %s total records' % (count)) return tumor_type2cghub_records
def create(self, platform, variant, output_dir): if self.is_created: log_error('framework {} has already been created'.format(self.name)) return self log_info('creating framework: {}'.format(self.name)) self.framework_path = '{}/{}/{}/{}.framework'.format( output_dir, platform.full_name, variant.name, self.name) self.bin_path = '{}/{}'.format(self.framework_path, self.name) # set the framework path and make our dylib.framework # directory os.makedirs(self.framework_path) self.abs_path = os.path.abspath(self.framework_path) # copy the binary to the new path copy2(self.library_path, self.bin_path) # fix run paths, since we will be # encapsulating each dylib if self.should_fix_rpaths: self.__fix_rpaths() self.is_created = True return self
def process_cghub(config, type_uri = 'detail', log = None, removedups = False, limit = -1, verbose = False, print_response = False): """ return type: tumor_type2cghub_records: organizes the cghub record classes per tumor type """ log_info(log, 'begin process cghub') module = import_module(config['cghub_module']) mappings = config['metadata_locations']['cghub'] cghub_records, _ = module.main(mappings['study'], log = log, removedups = removedups, limit = limit) tumor_type2cghub_records = {} count = 0 seen_bad_codes = set() for cghub_record in cghub_records: if 0 == count % 8192: log_info(log, '\tprocess %s cghub records' % (count)) count += 1 tumor_type2cghub_records.setdefault(cghub_record.disease_abbr, []).append(create_cghub_metadata(mappings, cghub_record, seen_bad_codes, log)) log_info(log, 'finished process cghub: %s total records' % (count)) return tumor_type2cghub_records
def statistics(log, filename2cghubRecords, minmaxsize, verbose): states = {} centers = {} studies = {} sampleIDs = [{}, {}, {}, {}, {}, {}, {}, {}] diseases = {} analyte_codes = {} sample_types = {} strategies = {} platforms = {} refassems = {} models = {} for record in filename2cghubRecords.itervalues(): states[record.state] = states.setdefault(record.state, 0) + 1 centers[record.center_name] = centers.setdefault(record.center_name, 0) + 1 studies[record.study] = studies.setdefault(record.study, 0) + 1 diseases[record.disease_abbr] = diseases.setdefault(record.disease_abbr, 0) + 1 analyte_codes[record.analyte_code] = analyte_codes.setdefault(record.analyte_code, 0) + 1 sample_types[record.sample_type] = sample_types.setdefault(record.sample_type, 0) + 1 strategies[record.library_strategy] = strategies.setdefault(record.library_strategy, 0) + 1 platforms[record.platform] = platforms.setdefault(record.platform, 0) + 1 refassems[record.refassem_short_name] = refassems.setdefault(record.refassem_short_name, 0) + 1 models[record.platform_full_name] = models.setdefault(record.platform_full_name, 0) + 1 try: fields = record.legacy_sample_id.split('-') for index, field in enumerate(fields[:-3]): sampleIDs[index][field] = sampleIDs[index].setdefault(field, 0) + 1 for index, field in enumerate(fields[-3:]): sampleIDs[index + len(fields[:-3])][field] = sampleIDs[index + len(fields[:-3])].setdefault(field, 0) + 1 except: util.log_exception(log, 'problem splitting %s(%s:%s)' % (record.legacy_sample_id, index, field)) util.log_info(log, '\nStates') count = 0 for state, value in states.iteritems(): if count < 15: count += 1 util.log_info(log, '%s: %s' % (state, value)) else: util.log_info(log, '\t(of %s)' % (len(states.keys()))) break util.log_info(log, '') util.log_info(log, 'Centers') count = 0 for center, value in centers.iteritems(): if count < 15: count += 1 util.log_info(log, '%s: %s' % (center, value)) else: util.log_info(log, ' (of %s)' % (len(centers.keys()))) break util.log_info(log, '') util.log_info(log, 'Studies') count = 0 for studie, value in studies.iteritems(): if count < 15: count += 1 util.log_info(log, '%s: %s' % (studie, value)) else: util.log_info(log, '(of %s)' % (len(studies.keys()))) break util.log_info(log, '') if verbose: util.log_info(log, 'Sample ids:') count = 0 for sampleMap in sampleIDs: util.log_info(log, 'next part:') for sampleID, value in sampleMap.iteritems(): if count < 15: count += 1 util.log_info(log, '%s: %s' % (sampleID, value)) else: util.log_info(log, '(of %s)' % (len(sampleMap.keys()))) break util.log_info(log, '') count = 0 util.log_info(log, 'Diseases:') count = 0 for disease, value in diseases.iteritems(): count += 1 util.log_info(log, '%s: %s' % (disease, value)) util.log_info(log, '') util.log_info(log, 'Analyte codes:') count = 0 for analyte_code, value in analyte_codes.iteritems(): if count < 15: count += 1 util.log_info(log, '%s: %s' % (analyte_code, value)) else: util.log_info(log, '(of %s)' % (len(analyte_codes.keys()))) break util.log_info(log, '') util.log_info(log, 'Sample types') count = 0 for sample_type, value in sample_types.iteritems(): if count < 15: count += 1 util.log_info(log, '%s: %s' % (sample_type, value)) else: util.log_info(log, '(of %s)' % (len(sample_types.keys()))) break util.log_info(log, '') util.log_info(log, 'Strategies:') count = 0 for strategie, value in strategies.iteritems(): if count < 15: count += 1 util.log_info(log, '%s: %s' % (strategie, value)) else: util.log_info(log, '(of %s)' % (len(strategies.keys()))) break util.log_info(log, '') util.log_info(log, 'Platforms:') count = 0 for platform, value in platforms.iteritems(): if count < 15: count += 1 util.log_info(log, '%s: %s' % (platform, value)) else: util.log_info(log, '(of %s)' % (len(platforms.keys()))) break util.log_info(log, '') util.log_info(log, 'Reference Assembles:') count = 0 for refassem, value in refassems.iteritems(): if count < 15: count += 1 util.log_info(log, '%s: %s' % (refassem, value)) else: util.log_info(log, '(of %s)' % (len(refassems.keys()))) break util.log_info(log, '') util.log_info(log, 'Models:') count = 0 for model, value in models.iteritems(): if count < 15: count += 1 util.log_info(log, '%s: %s' % (model, value)) else: util.log_info(log, '(of %s)' % (len(models.keys()))) break util.log_info(log, '') util.log_info(log, '\n\t\tmax: %s\n\t\tmin: %s' % (minmaxsize['max'].write(), minmaxsize['min'].write()))
def main(platform, type_uri = 'detail', log = None, removedups = False, limit = -1, verbose = False, print_response = False): util.log_info(log, 'begin reading cghub archive') filename2cghubRecords = {} minmaxsize = {'min': CGHubFileInfo('', 500000000000, ''), 'max': CGHubFileInfo('', 1, '')} try: # archives = util.getURLData(manifest_uri, 'latest_manifest', log) response = urllib.urlopen(manifest_uri) archives = response.read() lines = archives.split('\n') util.log_info(log, '\tarchive size is %s with %s lines' % (len(archives), len(lines))) util.log_info(log, '\n\t' + '\n\t'.join(lines[:10])) except Exception as e: util.log_exception(log, 'problem fetching latest_manifest: %s') raise e headers = lines[0].split('\t') column_index2header = {} for index, header in enumerate(headers): if header in header2record_index.keys(): column_index2header[index] = header count = 0 dupcount = 0 for line in lines[1:]: if not line: continue if 0 == count % 4096: util.log_info(log, 'processed %s records' % (count)) count += 1 fields = line.split('\t') header2record = {} try: for index in column_index2header.keys(): header2record[header2record_index[column_index2header[index]]] = fields[index] except Exception as e: util.log_info(log, 'problem with parsing line(%s): %s' % (count, line)) if platform not in header2record[CGHubRecordInfo.study_index]: continue header2record.update(index2none) record = CGHubRecordInfo(header2record) filename = header2record[CGHubRecordInfo.bamFilename_index] if removedups and filename in filename2cghubRecords: if 'Live' == header2record[CGHubRecordInfo.state_index]: dupcount += 1 # check the dates and keep the latest currentdate = createDateTime(filename2cghubRecords[filename].upload_date) newdate = createDateTime(record.upload_date) if currentdate < newdate: filename2cghubRecords[filename] = record else: filename2cghubRecords[filename] = record if 'Live' == record.state: if minmaxsize['min'].filesize > record.files['bam'].filesize and record.files['bam'].filesize: minmaxsize['min'] = record.files['bam'] if minmaxsize['max'].filesize < record.files['bam'].filesize: minmaxsize['max'] = record.files['bam'] if not record.files['bam'].filesize: util.log_info(log, 'no file size: %s--%s' % (record.write(), record.files['bam'].write())) statistics(log, filename2cghubRecords, minmaxsize, verbose) util.log_info(log, 'finished reading cghub archive. %s total records, %s duplicates' % (count, dupcount)) return filename2cghubRecords.values(), minmaxsize