def __str__(self): s = "" s += "Filename: " + xstr(self.filename) + "\n" s += "NetworkData: \n" + xstr(self.networkData) + "\n" s += "Seed: " + xstr(self.seed) + "\n" s += "Time: " + xstr(self.time) + "\n" return s
def returnParametersAsList(self, groupid, ignore_aggregates=False): parameterList = "" counter = 1 # Traverse through each parameter in object's list for param in self.parameters: new_param = utils.xstr(self.name + "." + param) is_changed = False if self.ignore_aggregates == False: # Apply any functions to the parameter, ie SUM, AVG, etc. new_param, alias, is_changed = export_utils.apply_selection_function( utils.xstr(self.name + "." + param), groupid ) # For subqueries it is nessesary to create an alias for the aggregation function if is_changed: new_param += " as " + param # Concatenate to parameterList and add a comma if not the end of the list parameterList += new_param if counter != len(self.parameters): parameterList += "," counter += 1 if "uuid" not in self.parameters: if parameterList is not "": parameterList += "," + self.name + ".uuid" else: parameterList += self.name + ".uuid" return parameterList
def printStats(self): for idx, corpus in enumerate(self.corpus): d = (idx, utils.shortSeed(corpus.seed), utils.xstr(self._getIdxOfCorpus(corpus._parent)), utils.xstr(corpus.networkData.getFuzzMessageIndex()), corpus.stats["new"], corpus.stats["crashes"]) print " Corpus %3d (%5s): Parent: %3s Msg: %3s Children: %3d Crashes: %3d" % d
def _build(self): """ Build filter accoriding to filtering parameters. :rtype: list """ opts = self.opts filter_list = [] # No timestring parameter, range parameters a given if not opts.timestring and any( (xstr(opts.newer_than), xstr(opts.older_than))): print 'ERROR: Parameters newer_than/older_than require timestring to be given' sys.exit(1) # Timestring used alone without newer_than/older_than if opts.timestring is not None and not all( (xstr(opts.newer_than), xstr(opts.older_than))): f = api.filter.build_filter(kindOf='timestring', value=opts.timestring) if f: filter_list.append(f) # Timebase filtering timebased = zip(('newer_than', 'older_than'), (opts.newer_than, opts.older_than)) for opt, value in timebased: if value is None: continue f = api.filter.build_filter(kindOf=opt, value=value, timestring=opts.timestring, time_unit=opts.time_unit) if f: filter_list.append(f) # Add filtering based on suffix|prefix|regex patternbased = zip(('suffix', 'prefix', 'regex'), (opts.suffix, opts.prefix, opts.regex)) for opt, value in patternbased: if value is None: continue f = api.filter.build_filter(kindOf=opt, value=value) if f: filter_list.append(f) # Add exclude filter for pattern in opts.exclude or []: f = {'pattern': pattern, 'exclude': True} filter_list.append(f) return filter_list
def _build(self): """ Build filter accoriding to filtering parameters. :rtype: list """ opts = self.opts filter_list = [] # No timestring parameter, range parameters a given if not opts.timestring and any((xstr(opts.newer_than), xstr(opts.older_than))): print 'ERROR: Parameters newer_than/older_than require timestring to be given' sys.exit(1) # Timestring used alone without newer_than/older_than if opts.timestring is not None and not all((xstr(opts.newer_than), xstr(opts.older_than))): f = api.filter.build_filter(kindOf='timestring', value=opts.timestring) if f: filter_list.append(f) # Timebase filtering timebased = zip(('newer_than', 'older_than'), (opts.newer_than, opts.older_than)) for opt, value in timebased: if value is None: continue f = api.filter.build_filter(kindOf=opt, value=value, timestring=opts.timestring, time_unit=opts.time_unit) if f: filter_list.append(f) # Add filtering based on suffix|prefix|regex patternbased = zip(('suffix', 'prefix', 'regex'), (opts.suffix, opts.prefix, opts.regex)) for opt, value in patternbased: if value is None: continue f = api.filter.build_filter(kindOf=opt, value=value) if f: filter_list.append(f) # Add exclude filter for pattern in opts.exclude or []: f = {'pattern': pattern, 'exclude': True} filter_list.append(f) return filter_list
def _apply_filters(self, source_items, act_on): """Applies filters to a list of indices or snapshots. :param source_items: List of indices or snapshots. :param act_on: Specifies whether we act on indices or snapshots. """ opts = self.opts all_items_selected = opts.get('all_{0}'.format(act_on), None) # Choose explicitly chosen indices or snapshots # if act_on == 'indices': explicit_items = opts.index or [] else: explicit_items = opts.snapshot or [] # I don't care about using only timestring if it's a `dry_run` of show if not any((xstr(opts.newer_than), xstr(opts.older_than), opts.dry_run)) and \ opts.timestring: logger.warn('Used only timestring parameter.') logger.warn('Actions can be performed on all %s matching %s', act_on, opts.timestring) logger.debug("Full list of %s: %s", act_on, source_items) if not source_items: print 'ERROR. No {0} found in Elasticsearch.'.format(act_on) sys.exit(1) else: working_list = source_items # No filters has been added and not all items selected, # this means index or snapshot parameter is used alone. if not all_items_selected and not self.ifilter.filter_list: working_list = [] else: # Otherwise safely apply filtering working_list = self.ifilter.apply(working_list, act_on=act_on) # Include explict items into resulting working list. if explicit_items: working_list.extend( (i for i in explicit_items if i in source_items)) if not working_list: logger.error('No %s matched provided args: %s', act_on, opts) print "ERROR. No {} found in Elasticsearch.".format(act_on) sys.exit(99) # Make a sorted, unique list of indices/snapshots return sorted(list(set(working_list)))
def _apply_filters(self, source_items, act_on): """Applies filters to a list of indices or snapshots. :param source_items: List of indices or snapshots. :param act_on: Specifies whether we act on indices or snapshots. """ opts = self.opts all_items_selected = opts.get('all_{0}'.format(act_on), None) # Choose explicitly chosen indices or snapshots # if act_on == 'indices': explicit_items = opts.index or [] else: explicit_items = opts.snapshot or [] # I don't care about using only timestring if it's a `dry_run` of show if not any((xstr(opts.newer_than), xstr(opts.older_than), opts.dry_run)) and \ opts.timestring: logger.warn('Used only timestring parameter.') logger.warn('Actions can be performed on all %s matching %s', act_on, opts.timestring) logger.debug("Full list of %s: %s", act_on, source_items) if not source_items: print 'ERROR. No {0} found in Elasticsearch.'.format(act_on) sys.exit(1) else: working_list = source_items # No filters has been added and not all items selected, # this means index or snapshot parameter is used alone. if not all_items_selected and not self.ifilter.filter_list: working_list = [] else: # Otherwise safely apply filtering working_list = self.ifilter.apply(working_list, act_on=act_on) # Include explict items into resulting working list. if explicit_items: working_list.extend((i for i in explicit_items if i in source_items)) if not working_list: logger.error('No %s matched provided args: %s', act_on, opts) print "ERROR. No {} found in Elasticsearch.".format(act_on) sys.exit(99) # Make a sorted, unique list of indices/snapshots return sorted(list(set(working_list)))
def buildSubQuery(self, groupid, range=[None, time.strftime("%Y-%m-%d")], ignore_aggregates=False): # Parameters used in Subquery select Clause subParameters = self.returnParametersAsList(groupid) subGroupBy = self.returnGroupByAsList(groupid) # Required parameters for a subquery to do joins (date ranges are done internally) # Build Date Ranges for the where clause # Check DATES are valid, if not use default if not utils.is_valid_date(utils.xstr(range[0])): range[0] = "" if not utils.is_valid_date(utils.xstr(range[1])): range[1] = time.strftime("%Y-%m-%d") subDateRange = ( self.name + ".ds>='" + utils.xstr(range[0]).split(" ")[0] + "' and " + self.name + ".ds<='" + utils.xstr(range[1]).split(" ")[0] + "'" ) # Check if DATES have time, if so append server_date. if utils.has_valid_time(utils.xstr(range[0])): subDateRange += " and " + self.name + ".server_date>='" + utils.xstr(range[0]) + "'" if utils.has_valid_time(utils.xstr(range[1])): subDateRange += " and " + self.name + ".server_date<='" + utils.xstr(range[1]) + "'" subWhere = " where " + subDateRange if len(self.wheres) > 0: logicOp = " and " """for where in self.wheres: subWhere += logicOp subWhere += where logicOp = " or " """ for key in self.wheresMap.keys(): subWhere += logicOp counter = 1 subWhere += "(" for where in self.wheresMap[key]: logicOp = " or " subWhere += where if counter != len(self.wheresMap[key]): subWhere += logicOp counter += 1 subWhere += ")" logicOp = " and " query = "select " + subParameters + " from " + utils.xstr(self.name) + subWhere # if ignore aggregates is not nessesary then include the groupby parameters if self.ignore_aggregates == False: query = query + " group by " + subGroupBy return query
def raw_data(self): return '%s$%s$%s$%s$%s$%s$%s$%s$%s' % ( self.name, utils.xstr(self.timeout), utils.xstr(self.begin_time), utils.xstr(self.end_time), utils.xstr(self.begin_date), utils.xstr(self.end_date), utils.xstr(self.begin_lock_date), utils.xstr(self.end_lock_date), utils.xstr(self.day_mask), )
def __parseQueryDateRanges(self,range): if not utils.is_valid_date(utils.xstr(range[0])): range[0] = '' if not utils.is_valid_date(utils.xstr(range[1])): range[1] = time.strftime('%Y-%m-%d') if utils.has_valid_time(utils.xstr(range[0])): start_date, start_time = range[0].split(" ") else: start_date = range[0] start_time = '' if utils.has_valid_time(utils.xstr(range[1])): end_date, end_time = range[1].split(" ") else: end_date = range[1] end_time = '23:59:59' query = self.query query = query.replace("#STARTDATE",start_date) query = query.replace("#STARTTIME",start_time) query = query.replace("#ENDDATE",end_date) query = query.replace("#ENDTIME",end_time) self.query = query
def to_string(self): return "{ objectid: [" + xstr(self.objectid) + \ "], name: [" + xstr(self.name) + \ "], year: [" + xstr(self.year) + \ "], designer: [" + ','.join(self.designers) + \ "], publisher: [" + ','.join(self.publishers) + \ "], playtime: (" + "-".join(self.playtime) + \ "), players: (" + "-".join(self.players) + \ "), suggested: [" + xstr(self.suggested) + \ "], age: [" + xstr(self.age) + \ "], language: [" + xstr(self.language) + \ "], image: [" + xstr(self.image) + \ "], description: [" + xstr(self.description) + "] " + \ "}"
def main(input_fc=None, output_csv=None, mode=settings.mode): if not arcpy.Exists(input_fc): utils.msg("Input, {} doesn't exist.".format(input_fc)) sys.exit() try: with open(output_csv, 'wb') as output_file: writer = csv.writer(output_file, dialect='excel', \ quotechar='"', quoting=csv.QUOTE_ALL) # fields generated by ArcGIS & lacking a direct use in Shepherd ignored_fields = ['OID', 'OBJECTID', 'Shape'] # header row should contain our fields declaration fields = [ field.name for field in arcpy.ListFields(input_fc) if field.name not in ignored_fields ] writer.writerow(fields) our_date = "Date_formatted" date_exists = False # If we don't have a formatted date field, don't try to manipulate it. if our_date in fields: date_pos = fields.index(our_date) date_exists = True with arcpy.da.SearchCursor(input_fc, fields) as cursor: for row in cursor: formatted_row = list(row[:]) if date_exists: date_value = row[date_pos] try: formatted_row[date_pos] = formatDate(date_value) except Exception as e: arcpy.AddError(e) writer.writerow([utils.xstr(r) for r in formatted_row]) except Exception as e: utils.msg("Error creating output file.", mtype='error', exception=e) sys.exit() utils.msg("Exported results saved to %s." % output_csv) if mode == 'toolbox': time.sleep(4)
def main(input_fc=None, output_csv=None, mode=settings.mode): if not arcpy.Exists(input_fc): utils.msg("Input, {} doesn't exist.".format(input_fc)) sys.exit() try: with open(output_csv, 'wb') as output_file: writer = csv.writer(output_file, dialect='excel', \ quotechar='"', quoting=csv.QUOTE_ALL) # fields generated by ArcGIS & lacking a direct use in Shepherd ignored_fields = ['OID', 'OBJECTID', 'Shape'] # header row should contain our fields declaration fields = [field.name for field in arcpy.ListFields(input_fc) if field.name not in ignored_fields] writer.writerow(fields) our_date = "Date_formatted" date_exists = False # If we don't have a formatted date field, don't try to manipulate it. if our_date in fields: date_pos = fields.index(our_date) date_exists = True with arcpy.da.SearchCursor(input_fc, fields) as cursor: for row in cursor: formatted_row = list(row[:]) if date_exists: date_value = row[date_pos] try: formatted_row[date_pos] = formatDate(date_value) except Exception as e: arcpy.AddError(e) writer.writerow([utils.xstr(r) for r in formatted_row]) except Exception as e: utils.msg("Error creating output file.", mtype='error', exception=e) sys.exit() utils.msg("Exported results saved to %s." % output_csv) if mode == 'toolbox': time.sleep(4)
def returnGroupByAsList(self, groupid, ignore_aggregates=False): groupbyList = "" # Traverse through each parameter in object's list for param in self.parameters: is_changed = True if self.ignore_aggregates == False: # Apply any functions to the parameter, ie SUM, AVG, etc. is_changed = export_utils.apply_selection_function(utils.xstr(self.name + "." + param), groupid)[2] # If no change was made by function, then it belongs in the groupby list if not is_changed and param not in self.groupByList: self.groupByList.append(param) # Groupby Required Parameters - make sure they are there if "uuid" not in self.groupByList: self.groupByList.append("uuid") # Traverse through the new groupby list, this is the original parameter list, without parameters with math functions counter = 1 for groupby_param in self.groupByList: groupbyList += self.name + "." + groupby_param if counter != len(self.groupByList): groupbyList += "," counter += 1 return groupbyList
def main(input_features=None, id_field=None, where_clause=None, output_coords=None, output_genetics=None, mode=settings.mode): # get the spatial reference of our input, determine the type desc = arcpy.Describe(input_features) sr = desc.spatialReference if sr.type not in ['Geographic', 'Projected']: utils.msg("This tools only works with geographic or projected data.", mtype='error') sys.exit() if not id_field: id_field = settings.id_field # Find our Loci columns. loci = utils.Loci(input_features) utils.msg("loci set: {0}".format(",".join(loci.names))) comments = """Export to Alleles in Space from the input `{input_features}`. Export occurred on {datetime} Coordinates are in {sr_name}, a {sr_type} coordinate system.""".format( input_features=input_features, datetime=datetime.now(), \ sr_name=sr.name, sr_type=sr.type.lower()) if sr.type == 'Geographic': # geographic data expected to be (lat, lon) loc_a = settings.y_coord loc_b = settings.x_coord else: # two coordinates in a projected space loc_a = settings.x_coord loc_b = settings.y_coord selected_columns = [id_field, loc_a, loc_b] + loci.columns rows = arcpy.da.SearchCursor(input_features, selected_columns, where_clause) coordinate_rows = [] genetic_rows = [] for row in rows: id_field = row[0] loc_a_val = row[1] loc_b_val = row[2] coordinate_row = [id_field, loc_a_val, loc_b_val] genetic_row = [id_field] for (key, cols) in loci.fields.items(): # loci values should be separated by an '\'. loci_values = [] for col in cols: col_pos = selected_columns.index(col) allele_val = row[col_pos] # handle missing and incomplete genotypes. if allele_val is None: allele_val = "0" else: allele_val = str(allele_val) loci_values.append(allele_val) genetic_row.append('\\'.join(loci_values)) coordinate_rows.append(coordinate_row) genetic_rows.append(genetic_row) try: # copy the final result back to disk. utils.msg("Writing results to disk...") # Alleles in Space expects comma-delimited outputs sep = "," with open(output_coords, 'wb') as coords_file: for raw_row in coordinate_rows: # convert all data to strings row = [utils.xstr(s) for s in raw_row] coords_file.write("{0}\n".format(sep.join(row))) coords_file.write(";\n") coords_file.write(comments) utils.msg("Exported coordinates saved to %s." % output_coords) with open(output_genetics, 'wb') as genetics_file: # start the file with the number or loci genetics_file.write("{0}\n".format(loci.count)) for raw_row in genetic_rows: # convert all data to strings row = [utils.xstr(s) for s in raw_row] genetics_file.write("{0}\n".format(sep.join(row))) genetics_file.write(";\n") genetics_file.write(comments) utils.msg("Exported genetics saved to %s." % output_genetics) except Exception as e: utils.msg("Error creating output file.", mtype='error', exception=e) sys.exit() if mode == 'toolbox': time.sleep(4)
def run_geodesic_gp(input_fc, unit_factor, output_matrix, row_count, is_spagedi): input_fc_mem = 'in_memory/input_fc' try: utils.msg("Copying features into memory...") arcpy.CopyFeatures_management(input_fc, input_fc_mem) except Exception as e: msg = "Unable to copy features into memory." utils.msg(msg, mtype='error', exception=e) sys.exit() # get the spatial reference of our input, determine the type desc = arcpy.Describe(input_fc_mem) sr = desc.spatialReference if sr.type not in ['Geographic', 'Projected']: msg = "This tools only works with geographic or projected data." utils.msg(msg, mtype='error') sys.exit() utils.msg("Finding all input points...") distance_matrix = OrderedDict() points = OrderedDict() records = arcpy.da.SearchCursor(input_fc_mem, ['OID@', 'SHAPE@XY']) for row in records: (fid, point) = row points[fid] = arcpy.Point(point[0], point[1]) indicator = 0 utils.msg("Computing distances...") for (fid, from_point) in points.items(): pct_progress = int(fid / float(row_count) * 100) if pct_progress > indicator: indicator = pct_progress utils.msg("{0}%".format(indicator)) p1 = points[fid] distance_matrix[fid] = OrderedDict() for (to_fid, to_point) in points.items(): if to_fid == fid: dist = 0 elif distance_matrix.has_key(to_fid) and \ distance_matrix[to_fid][fid] is not None: # here, modeling a symmetrical matrix dist = distance_matrix[to_fid][fid] else: p2 = points[to_fid] if p1.equals(p2): dist = 0 else: # Each Polyline initialization must pay the COM object # gods, and ends up making this much more expensive than # AO C++ or even comtypes calls. line = arcpy.Polyline(arcpy.Array([p1, p2]), sr) # distance, always returned in meters, scale by our # expected result units. dist = line.getLength("GEODESIC") * unit_factor distance_matrix[fid][to_fid] = dist utils.msg("Distance matrix calculations complete.") # FIXME: generate it as a CSV file, then do TableToTable to pull it back in # Now compute the lines between these locations. try: # copy the final result back to disk. utils.msg("Writing results to disk...") # The SPAGeDi matrix format are described in section 3.7 of the manual. if is_spagedi: first_header_cell = "M%i" % row_count sep = "\t" else: first_header_cell = "" sep = "," with open(output_matrix, 'w') as csv: # initialize with our header row output_rows = [[first_header_cell] + \ [str(s) for s in distance_matrix.keys()]] for (fid, row) in distance_matrix.items(): res = [str(fid)] + [utils.xstr(s) for s in row.values()] output_rows.append(res) for row in output_rows: csv.write("{0}\n".format(sep.join(row))) if is_spagedi: csv.write("END\n") except Exception as e: utils.msg("Error creating distance matrix.", mtype='error', exception=e) sys.exit()
def subreddit_entry_to_listitem(subreddit_entry): from utils import compose_list_item, build_script, xstr, prettify_reddit_query addtl_subr_info={} nsfw=False icon=banner=header=public_description=display_name=override_header_image=None header_ar=0 addtl_subr_info=ret_sub_info(subreddit_entry) #strip out the alias identifier from the subreddit string retrieved from the file so we can process it. entry_type, subreddit, alias, shortcut_description=parse_subreddit_entry(subreddit_entry) icon=default_icon=ret_settings_type_default_icon(entry_type) pretty_label=prettify_reddit_query(alias) pretty_label=pretty_label.replace('+',' + ') if addtl_subr_info: icon=addtl_subr_info.get('icon_img') banner=addtl_subr_info.get('banner_img') #rectangular shape header=addtl_subr_info.get('header_img') #square shape from bannerTvImageUrl header_ar=img_ar(addtl_subr_info.get('header_size')) if (header_ar > 8) and (not icon): #some header_img are very wide. this is to check and override the icon display in the gui override_header_image=header banner=header public_description=xstr( addtl_subr_info.get('public_description','')) display_name=xstr(addtl_subr_info.get('display_name','')) #log('{} icon={} header={} banner={}'.format( subreddit, repr(icon), repr(header), repr(banner) )) if entry_type=='link': #<-- added new ability to have youtube channels as a shortcut on the main screen #here, the subreddit variable contains a url. we made sure that it points to a youtube channel(ContextMenus.py). that way, there is no need to specify 'channel' when calling listRelatedVideo liz = compose_list_item( pretty_label, entry_type, "", "script", build_script("listRelatedVideo",subreddit,alias) ) else: #domain, subreddit, combined, search, multireddit reddit_url=assemble_reddit_filter_string("",subreddit, "yes") if entry_type=='domain': #remove the identifier that this setting is a domain pretty_label=re.findall(r'(?::|\/domain\/)(.+)',subreddit)[0] if subreddit.lower() in ["all","popular"]: liz = compose_list_item( pretty_label, entry_type, "", "script", build_script("listSubReddit",reddit_url,alias) ) else: if addtl_subr_info: #if we have additional info about this subreddit #log(repr(addtl_subr_info)) #title=addtl_subr_info.get('title','')+'\n' #display_name=xstr(addtl_subr_info.get('display_name','')) #if samealphabetic( title, display_name): title='' #header_title=xstr(addtl_subr_info.get('header_title','')) #in reddit_viewer, title, header_title and public_description is shown as plot nsfw=addtl_subr_info.get('over18') icon=next((item for item in [icon,banner,header] if item ), '') or default_icon #picks the first item that is not None #icon=icon or default_icon #log( pretty_label + ' icon=' + icon + ' nsfw='+repr(nsfw)) liz = compose_list_item( pretty_label, entry_type, "", "script", build_script("listSubReddit",reddit_url,alias) ) else: liz = compose_list_item( pretty_label, entry_type, "", "script", build_script("listSubReddit",reddit_url,alias) ) liz.setArt({ "thumb": icon, "banner":banner, "fanart":override_header_image }) liz.setInfo('video', {"Title":display_name, "plot":public_description} ) if nsfw: liz.setProperty('nsfw', 'true' ) return liz
def main(input_features=None, where_clause=None, order_by=None, output_name=None, mode=settings.mode): # get the spatial reference of our input, determine the type desc = arcpy.Describe(input_features) sr = desc.spatialReference if sr.type not in ['Geographic', 'Projected']: utils.msg("This tools only works with geographic or projected data.", mtype='error') sys.exit() # Find our Loci columns. loci = utils.Loci(input_features) utils.msg("loci set: {0}".format(",".join(loci.names))) # sql clause can be prefix or suffix; set up ORDER BY sql_clause = (None, "ORDER BY {0} ASC".format(order_by)) pops = OrderedDict() # query the input_features in ascending order; filtering as needed rows = arcpy.da.SearchCursor(input_features, order_by, where_clause, "", "", sql_clause) row_count = 0 for row in rows: row_count += 1 pop = row[0] if pops.has_key(pop): pops[pop] += 1 else: pops[pop] = 1 # Start with any number of header lines describing what this # file is, each line should be prefixed with //. comments = """// Export to SPAGeDi from the input `{input_features}`. // Export occurred on {datetime} // Coordinates are in {sr_name}, a {sr_type} coordinate system.""".format( input_features=input_features, datetime=datetime.now(), \ sr_name=sr.name, sr_type=sr.type) comment_row = [comments] """ There are three possible ways to specify population (SPAGeDi manual, 2.4): 1) as categorical groups, where one population includes all individuals sharing the same categorical variable. 2) as spatial groups, where a spatial group includes all individuals sharing the same spatial coordinates and following each other in the data file. 3) as spatio-categorical groups, where a spatio-categorical group includes all individuals belonging to both the same spatial group and categorical group. When populations are defined using the categorical variable, the spatial coordinates of a given population are computed by averaging the coordinates of the individuals it contains. """ # FIXME: another spot where 'observation_id' differs from 'individual_id'; # deduplicate in order to have one value PER individual PER population. categories = len(pops.keys()) # FIXME: presumes only two coords, SPAGeDi supports three. Extract depth? if sr.type == 'Geographic': # Assumes decimal degrees; based on the statement 'if the number of # spatial coordinates is set to -2, latitudes and longitudes must be # given in degrees with decimal, using negative numbers for Southern # latitude or Western longitude. xy_type = -2 # geographic data expected to be (lat, lon) loc_a = settings.y_coord loc_b = settings.x_coord else: # two coordinates in a projected space xy_type = 2 loc_a = settings.x_coord loc_b = settings.y_coord num_loci = loci.count # FIXME: loci number of digits, defined in SPAGeDi manual 3.1 as "number # of digits used to code one allele (1 to 3); or set a value =0 (in fact # the value given for missing data) in the case of dominant markers". loci_digits = 2 # get the maximum number of different values per loci max_ploidy = max(map(len, loci.fields.values())) # 1st line: set of 6 numbers separated by a tabulation representing: header_row = [ row_count, # number of invidivuals categories, # number of categories xy_type, # number of coordinates num_loci, # number of loci loci_digits, # number of digits used to code one allele max_ploidy # max ploidy in the data ] # 2nd line: # of distance intervals; upper distance of each interval. # note 1: alternatively you can enter only the desired number of intervals # preceded by a negative sign; the program then defines the # n maximal distances in such a way that the number of pairwise # comparisons within each distance interval is approx. constant. # note 2: if you do not wish distance intervals, put 0 # note 3: if you use latitude + longitude, distance intervals # must be given in km. # TODO: what is this and how do we use it effectively? distances_row = [0] # 3rd line: column labels (<=15 characters). base_cols = [settings.id_field, order_by, loc_a, loc_b] labels_row = base_cols + loci.columns # where_clause is used to ensure only those records with genetic data # are copied to the output. selected_columns = base_cols + loci.columns rows = arcpy.da.SearchCursor(input_features, selected_columns, \ where_clause, "", "", sql_clause) data_rows = [] for row in rows: # our two string fields can't contain spaces, based on Autocio.c: 3857 id_field = str(row[0]).replace(" ", "_") pop_field = str(row[1]).replace( " ", "_") # 'order_by', or population 'group by' loc_a_val = row[2] loc_b_val = row[3] data_row = [id_field, pop_field, loc_a_val, loc_b_val] for (key, cols) in loci.fields.items(): # Loci data can be encoded in a number of formats, the values # separated by any non-numeric values (SPAGeDi manual, 3.2.1). # Here, we use spaces. loci_values = [] for col in cols: col_pos = selected_columns.index(col) allele_val = row[col_pos] # FIXME: handle missing and incomplete genotypes. if allele_val is None: allele_val = "0" else: allele_val = str(allele_val) loci_len = len(allele_val) if loci_len > loci_digits: loci_digits = loci_len loci_values.append(allele_val) data_row.append(" ".join(loci_values)) data_rows.append(data_row) # update header based on revised loci_digits header_row[4] = loci_digits # Now compute the lines between these locations. try: # copy the final result back to disk. utils.msg("Writing results to disk...") output_rows = [comment_row, header_row, distances_row, labels_row] + \ data_rows # SPAGeDi expects tab-delimited outputs sep = "\t" with open(output_name, 'w') as output_file: for raw_row in output_rows: # convert all data to strings row = [utils.xstr(s) for s in raw_row] output_file.write("{0}\n".format(sep.join(row))) # after the last line of invidivual data the word END is required. output_file.write("END\n") except Exception as e: utils.msg("Error creating output file.", mtype='error', exception=e) sys.exit() utils.msg("Exported results saved to %s." % output_name) if mode == 'toolbox': time.sleep(4)
def export_report(parameters, keys = 'uuid', range = [None,time.strftime('%Y-%m-%d')], type = 'csv', outputname = None, emailaddr = None, where = None, description = 'No description', distinct = 'Yes', groupid = 'default', ftpuser = '******', vendor_event = 'InstallationInfo', report_name = None, debug = False, raw_output = False, scp = False, target_user = None, target_server = None, target_dir = None): try: COMMAND = '' #Build Query into COMMAND (Prefix: "hive --config $HIVE_HOME/conf -e \"" is added in function export_hive_to_file()) COMMAND = "\"use " + groupid + "; " if report_name is not None: reportBuilder = report_builder.ReportBuilder(report_name,range,groupid) COMMAND += reportBuilder.getReportQuery() else: #Create QueryBuilder Object queryBuilder = query_builder.QueryBuilder() #Sanitize Parameters #Creates: ###eventList - A List of Event Objects that have already been populated by sanatize parameters function. ###parameters - A List of the original parameters requested in it's original order. ###parameters_alias - A List of the aliases of the original parameters requested in it's order. eventList, parameters, parametersAliasList = export_utils.sanitize_parameters(parameters,groupid,raw_output) #Sanitize Where where, where_tables = export_utils.sanitize_where(where, parameters, groupid, eventList, vendor_event) #Add eventList to queryBuilder queryBuilder.addToEvents(eventList) COMMAND += queryBuilder.buildQuery(groupid,type,range,parameters,raw_output) COMMAND += "\"" print print "Query:" print "-------------------------" print COMMAND print "-------------------------" print if debug == False: print "START TIME: [" + time.strftime("%Y-%m-%d-%H:%M:%S", time.localtime()) + "]" print "START COMMAND: " + COMMAND outputpath, outputname, user, password = export_hive_to_file(COMMAND, utils.listToString(parametersAliasList), type, outputname, ftpuser) try: email("Report: Data Export Utility","complete!","smithmicro.com",emailaddr,"{'text':'email from export report', 'html':'<p>Your file is now complete. To access your report, click on this link: <a href=\"ftp://"+os.uname()[1]+"/"+type+"/"+outputname+"\">"+outputname+"</a> and enter your username and password below.</p><p>Username: <b>"+user.strip()+"</b><br />Password: <b>"+password.strip()+"</b></p>'}") except: print "The Email Server has Raised an Error" if bool(scp) == True: email_addr = user.strip() scp_method(email_addr, outputname, target_user, target_server, target_dir) except IOError, e: print "I/O error(%(error0)s):(%(error1)s)" % { "error0":e[0],"error1":e[1] } email("Export_Report","failed","analytics.smithmicro.com",emailaddr,"\nError:\nI/O error("+utils.xstr(e[0])+"): "+xstr(e[1])+"\n"+utils.xstr(sys.exc_info()[0]))
def subreddit_entry_to_listitem(subreddit_entry): from utils import compose_list_item, build_script, xstr, prettify_reddit_query addtl_subr_info = {} nsfw = False icon = banner = header = public_description = display_name = override_header_image = None header_ar = 0 addtl_subr_info = ret_sub_info(subreddit_entry) #strip out the alias identifier from the subreddit string retrieved from the file so we can process it. entry_type, subreddit, alias, shortcut_description = parse_subreddit_entry( subreddit_entry) icon = default_icon = ret_settings_type_default_icon(entry_type) pretty_label = prettify_reddit_query(alias) pretty_label = pretty_label.replace('+', ' + ') if addtl_subr_info: icon = addtl_subr_info.get('icon_img') banner = addtl_subr_info.get('banner_img') #rectangular shape header = addtl_subr_info.get( 'header_img') #square shape from bannerTvImageUrl header_ar = img_ar(addtl_subr_info.get('header_size')) if (header_ar > 8) and ( not icon ): #some header_img are very wide. this is to check and override the icon display in the gui override_header_image = header banner = header public_description = xstr(addtl_subr_info.get('public_description', '')) display_name = xstr(addtl_subr_info.get('display_name', '')) #log('{} icon={} header={} banner={}'.format( subreddit, repr(icon), repr(header), repr(banner) )) if entry_type == 'link': #<-- added new ability to have youtube channels as a shortcut on the main screen #here, the subreddit variable contains a url. we made sure that it points to a youtube channel(ContextMenus.py). that way, there is no need to specify 'channel' when calling listRelatedVideo liz = compose_list_item( pretty_label, entry_type, "", "script", build_script("listRelatedVideo", subreddit, alias)) else: #domain, subreddit, combined, search, multireddit reddit_url = assemble_reddit_filter_string("", subreddit, "yes") if entry_type == 'domain': #remove the identifier that this setting is a domain pretty_label = re.findall(r'(?::|\/domain\/)(.+)', subreddit)[0] if subreddit.lower() in ["all", "popular"]: liz = compose_list_item( pretty_label, entry_type, "", "script", build_script("listSubReddit", reddit_url, alias)) else: if addtl_subr_info: #if we have additional info about this subreddit #log(repr(addtl_subr_info)) #title=addtl_subr_info.get('title','')+'\n' #display_name=xstr(addtl_subr_info.get('display_name','')) #if samealphabetic( title, display_name): title='' #header_title=xstr(addtl_subr_info.get('header_title','')) #in reddit_viewer, title, header_title and public_description is shown as plot nsfw = addtl_subr_info.get('over18') icon = next( (item for item in [icon, banner, header] if item), '') or default_icon #picks the first item that is not None #icon=icon or default_icon #log( pretty_label + ' icon=' + icon + ' nsfw='+repr(nsfw)) liz = compose_list_item( pretty_label, entry_type, "", "script", build_script("listSubReddit", reddit_url, alias)) else: liz = compose_list_item( pretty_label, entry_type, "", "script", build_script("listSubReddit", reddit_url, alias)) liz.setArt({ "thumb": icon, "banner": banner, "fanart": override_header_image }) liz.setInfo('video', {"Title": display_name, "plot": public_description}) if nsfw: liz.setProperty('nsfw', 'true') return liz
def index(url,name,type_): from utils import xstr, samealphabetic, hassamealphabetic from reddit import load_subredditsFile, parse_subreddit_entry, create_default_subreddits, assemble_reddit_filter_string, ret_sub_info, ret_settings_type_default_icon if not os.path.exists(subredditsFile): #if not os.path.exists(subredditsFile): create_default_subreddits() subredditsFile_entries=load_subredditsFile() subredditsFile_entries.sort(key=lambda y: y.lower()) addtl_subr_info={} xbmcplugin.setContent(pluginhandle, "mixed") #files, songs, artists, albums, movies, tvshows, episodes, musicvideos next_mode='listSubReddit' for subreddit_entry in subredditsFile_entries: addtl_subr_info=ret_sub_info(subreddit_entry) entry_type, subreddit, alias, shortcut_description=parse_subreddit_entry(subreddit_entry) icon=default_icon='' #addon_path+"/resources/skins/Default/media/"+ret_settings_type_default_icon(entry_type) url= assemble_reddit_filter_string("",subreddit, "yes") if subreddit.lower() in ["all","popular"]: addDir(subreddit, url, next_mode, icon, subreddit, { "plot": translation(30009) } ) #Displays the currently most popular content from all of reddit else: if addtl_subr_info: #if we have additional info about this subreddit title=xstr(addtl_subr_info.get('title'))+'\n' display_name=xstr(addtl_subr_info.get('display_name')) if samealphabetic( title, display_name): title='' header_title=xstr(addtl_subr_info.get('header_title')) public_description=xstr( addtl_subr_info.get('public_description')) if samealphabetic( header_title, public_description): public_description='' if samealphabetic(title,public_description): public_description='' if entry_type=='subreddit': display_name='r/'+display_name shortcut_description='[COLOR cadetblue][B]%s[/B][/COLOR]\n%s[I]%s[/I]\n%s' %(display_name,title,header_title,public_description ) icon=addtl_subr_info.get('icon_img') banner=addtl_subr_info.get('banner_img') header=addtl_subr_info.get('header_img') #usually the small icon on upper left side on subreddit screen icon=next((item for item in [icon,banner,header] if item ), '') or default_icon addDirR(alias, url, next_mode, icon, type_=subreddit, listitem_infolabel={ "plot": shortcut_description }, file_entry=subreddit_entry, banner_image=banner ) else: addDirR(alias, url, next_mode, icon, subreddit, { "plot": shortcut_description }, subreddit_entry ) addDir("[B]- "+translation(30001)+"[/B]", "", 'addSubreddit', "", "", { "plot": translation(30006) } ) #"Customize this list with your favorite subreddit." addDir("[B]- "+translation(30005)+"[/B]", "",'searchReddits', "", "", { "plot": translation(30010) } ) #"Search reddit for a particular post or topic xbmcplugin.endOfDirectory(pluginhandle)
def main(input_features=None, id_field=None, where_clause='', order_by=None, output_name=None, format_type='Excel', mode='toolbox'): script_path = os.path.abspath(__file__) utils.msg("Executing {}...".format(script_path)) # try to set the id based on input, otherwise go off of the config. if id_field is not None: primary_id = id_field else: primary_id = settings.id_field # set mode based on how script is called. settings.mode = mode add_output = arcpy.env.addOutputsToMap arcpy.env.addOutputsToMap = True # ensure our order by field exists fields = [f.name for f in arcpy.ListFields(input_features)] if not order_by in fields: utils.msg("Unable to find order_by field, `{}`".format(order_by)) sys.exit() # The Input Feature Class # == input_features # Where clause that can be used to pull out only those rows with genetic # data from the feature class. # NOTE: This parameter is optional and was included because some data # sets may have individual IDs based on more than just genetics # (i.e. photo-id). If your data only has genetic records, this # parameter can be left blank. # == where_clause # The Attribute Field that distinguishes the populations in the input. # NOTE: This parameter is optional and was included because some data # sets may have more than one population in it. # == order_by try: # test opening the file to which the data will be written output_file = open(output_name, "wb") except Exception as e: utils.msg("Unable to open output file", mtype='error', exception=e) sys.exit() # initialize our haplotypes data haplotypes = utils.Haplotype(input_features) # set up the environment depending on the output format if format_type == 'Excel': # xlwt will write directly; only opened to make sure we can write to the location output_file.close() try: import xlwt except ImportError as e: msg = "Writing Excel Spreadsheets requires the `xlwt` library, which is" + \ " included in ArcGIS 10.2+. If you'd like Excel support in " + \ " ArcGIS 10.1, please install `xlwt` manually from PyPI: " + \ " https://pypi.python.org/pypi/xlwt/0.7.3" utils.msg(msg, mtype='error', exception=e) sys.exit() # only replace Nones, keep numeric types from utils import xrep as xstr from utils import zrep as zstr else: # convert all strings to text from utils import zstr as zstr from utils import xstr as xstr # list of lists to emulate the Excel sheet output_rows = [] haplotype_rows = [] utils.msg("Output file open and ready for data input") # Find our Loci columns. loci = utils.Loci(input_features) utils.msg("loci set: {0}".format(",".join(loci.names))) """ header row contains (in order): - number of loci - number of samples - number of populations - size of pop 1 - size of pop 2 - ... second row contains: - three blank cells - loci 1 label - loci 2 label - ... DATA starts at C4. See "GenAlEx Guide.pdf" page 15. """ # sql clause can be prefix or suffix; set up ORDER BY sql_clause = (None, "ORDER BY {0} ASC".format(order_by)) # query the input_features in ascending order; filtering as needed selected_columns = order_by pops = OrderedDict() rows = arcpy.da.SearchCursor(input_features, selected_columns, where_clause, "", "", sql_clause) row_count = 0 for row in rows: row_count += 1 pop = row[0] if pops.has_key(pop): pops[pop] +=1 else: pops[pop] = 1 pop_counts = [xstr(p) for p in pops.values()] # Creating the GenAlEx header information required for the text file. output_rows += [[loci.count, row_count, len(pops.keys())] + pop_counts] # optional title, then a list of each population output_rows += [['', '', ''] + pops.keys()] # first two rows almost exactly the same haplotype_rows = copy.deepcopy(output_rows) # 'number of loci' should be 1 for haplotype-only data haplotype_rows[0][0] = 1 loci_labels = [] for (key, cols) in loci.fields.items(): loci_labels += [key] + [''] * (len(cols) - 1) # get the spatial reference of our input, determine the type desc = arcpy.Describe(input_features) sr = desc.spatialReference if sr.type == 'Projected': loc_a = settings.x_coord loc_b = settings.y_coord if sr.type == 'Geographic': # geographic data expected to be (lat, lon) loc_a = settings.y_coord loc_b = settings.x_coord primary_columns = [primary_id, order_by, loc_a, loc_b] exclude = primary_columns + loci.columns + ['OBJECTID', 'Shape'] unselected_columns = [] for field in fields: # add any field not currently mapped if field not in exclude: unselected_columns.append(field) # extra fields should start with an empty line, the location, then any # columns not otherwise mapped. extra_columns = ['', loc_a, loc_b] + unselected_columns output_rows += [[primary_id, order_by] + loci_labels + extra_columns] if haplotypes.defined: haplotype_rows += [[primary_id, order_by, haplotypes.column]] utils.msg("Header info written to output") # Note the WhereClause: Because the SPLASH data has both photo-id and genetic # records, but GenAlEx only uses genetic data, the WhereClause is used to ensure # only those records with genetic data are copied to the text file. selected_columns = primary_columns + unselected_columns + loci.columns for row in arcpy.da.SearchCursor(input_features, selected_columns, where_clause, "", "", sql_clause): id_field = row[0] # as set on import pop = row[1] # second column is 'order_by', or key column loc_a_val = row[2] loc_b_val = row[3] unselected_rows = list(row[4:len(unselected_columns)+4]) result_row = [id_field, pop] for (key, cols) in loci.fields.items(): for col in cols: col_pos = selected_columns.index(col) result_row.append(row[col_pos]) result_row = result_row + ["", loc_a_val, loc_b_val] + unselected_rows output_rows += [[zstr(s) for s in result_row]] if haplotypes.defined: row_val = row[selected_columns.index(haplotypes.column)] if row_val: haplotype_val = haplotypes.lookup[row_val] else: haplotype_val = 0 haplotype_res = [id_field, pop, haplotype_val] haplotype_rows += [[zstr(s) for s in haplotype_res]] # depending on our driver, handle writing dependent on type if format_type == 'Excel': # initialize our spreadsheet workbook = xlwt.Workbook() # codominant data worksheet_co = workbook.add_sheet('Codominant') for (i, row) in enumerate(output_rows): for (j, val) in enumerate(row): worksheet_co.write(i, j, val) # the haplotype data if haplotypes.defined: worksheet_hap = workbook.add_sheet('Haplotype') for (i, row) in enumerate(haplotype_rows): for (j, val) in enumerate(row): worksheet_hap.write(i, j, val) # write out a mapping of haplotype names worksheet_hap_map = workbook.add_sheet('Haplotype Map') worksheet_hap_map.write(0, 0, haplotypes.column) worksheet_hap_map.write(0, 1, 'Numeric Code') for (i, (hap, code)) in enumerate(haplotypes.lookup.items(), start=1): worksheet_hap_map.write(i, 0, hap) worksheet_hap_map.write(i, 1, code) workbook.save(output_name) else: # create a CSV writer writer = csv.writer(output_file, dialect='excel', quotechar='"', quoting=csv.QUOTE_ALL) for row in output_rows: writer.writerow(row) utils.msg("Exported results saved to %s." % output_name) # Close Output text file output_file.close() if mode == 'toolbox': time.sleep(4) arcpy.env.addOutputsToMap = add_output
def main(input_features=None, id_field=None, where_clause='', order_by=None, output_name=None, format_type='Excel', mode='toolbox'): script_path = os.path.abspath(__file__) utils.msg("Executing {}...".format(script_path)) # try to set the id based on input, otherwise go off of the config. if id_field is not None: primary_id = id_field else: primary_id = settings.id_field # set mode based on how script is called. settings.mode = mode add_output = arcpy.env.addOutputsToMap arcpy.env.addOutputsToMap = True # ensure our order by field exists fields = [f.name for f in arcpy.ListFields(input_features)] if not order_by in fields: utils.msg("Unable to find order_by field, `{}`".format(order_by)) sys.exit() # The Input Feature Class # == input_features # Where clause that can be used to pull out only those rows with genetic # data from the feature class. # NOTE: This parameter is optional and was included because some data # sets may have individual IDs based on more than just genetics # (i.e. photo-id). If your data only has genetic records, this # parameter can be left blank. # == where_clause # The Attribute Field that distinguishes the populations in the input. # NOTE: This parameter is optional and was included because some data # sets may have more than one population in it. # == order_by try: # test opening the file to which the data will be written output_file = open(output_name, "wb") except Exception as e: utils.msg("Unable to open output file", mtype='error', exception=e) sys.exit() # initialize our haplotypes data haplotypes = utils.Haplotype(input_features) # set up the environment depending on the output format if format_type == 'Excel': # xlwt will write directly; only opened to make sure we can write to the location output_file.close() try: import xlwt except ImportError as e: msg = "Writing Excel Spreadsheets requires the `xlwt` library, which is" + \ " included in ArcGIS 10.2+. If you'd like Excel support in " + \ " ArcGIS 10.1, please install `xlwt` manually from PyPI: " + \ " https://pypi.python.org/pypi/xlwt/0.7.3" utils.msg(msg, mtype='error', exception=e) sys.exit() # only replace Nones, keep numeric types from utils import xrep as xstr from utils import zrep as zstr else: # convert all strings to text from utils import zstr as zstr from utils import xstr as xstr # list of lists to emulate the Excel sheet output_rows = [] haplotype_rows = [] utils.msg("Output file open and ready for data input") # Find our Loci columns. loci = utils.Loci(input_features) utils.msg("loci set: {0}".format(",".join(loci.names))) """ header row contains (in order): - number of loci - number of samples - number of populations - size of pop 1 - size of pop 2 - ... second row contains: - three blank cells - loci 1 label - loci 2 label - ... DATA starts at C4. See "GenAlEx Guide.pdf" page 15. """ # sql clause can be prefix or suffix; set up ORDER BY sql_clause = (None, "ORDER BY {0} ASC".format(order_by)) # query the input_features in ascending order; filtering as needed selected_columns = order_by pops = OrderedDict() rows = arcpy.da.SearchCursor(input_features, selected_columns, where_clause, "", "", sql_clause) row_count = 0 for row in rows: row_count += 1 pop = row[0] if pops.has_key(pop): pops[pop] += 1 else: pops[pop] = 1 pop_counts = [xstr(p) for p in pops.values()] # Creating the GenAlEx header information required for the text file. output_rows += [[loci.count, row_count, len(pops.keys())] + pop_counts] # optional title, then a list of each population output_rows += [['', '', ''] + pops.keys()] # first two rows almost exactly the same haplotype_rows = copy.deepcopy(output_rows) # 'number of loci' should be 1 for haplotype-only data haplotype_rows[0][0] = 1 loci_labels = [] for (key, cols) in loci.fields.items(): loci_labels += [key] + [''] * (len(cols) - 1) # get the spatial reference of our input, determine the type desc = arcpy.Describe(input_features) sr = desc.spatialReference if sr.type == 'Projected': loc_a = settings.x_coord loc_b = settings.y_coord if sr.type == 'Geographic': # geographic data expected to be (lat, lon) loc_a = settings.y_coord loc_b = settings.x_coord primary_columns = [primary_id, order_by, loc_a, loc_b] exclude = primary_columns + loci.columns + ['OBJECTID', 'Shape'] unselected_columns = [] for field in fields: # add any field not currently mapped if field not in exclude: unselected_columns.append(field) # extra fields should start with an empty line, the location, then any # columns not otherwise mapped. extra_columns = ['', loc_a, loc_b] + unselected_columns output_rows += [[primary_id, order_by] + loci_labels + extra_columns] if haplotypes.defined: haplotype_rows += [[primary_id, order_by, haplotypes.column]] utils.msg("Header info written to output") # Note the WhereClause: Because the SPLASH data has both photo-id and genetic # records, but GenAlEx only uses genetic data, the WhereClause is used to ensure # only those records with genetic data are copied to the text file. selected_columns = primary_columns + unselected_columns + loci.columns for row in arcpy.da.SearchCursor(input_features, selected_columns, where_clause, "", "", sql_clause): id_field = row[0] # as set on import pop = row[1] # second column is 'order_by', or key column loc_a_val = row[2] loc_b_val = row[3] unselected_rows = list(row[4:len(unselected_columns) + 4]) result_row = [id_field, pop] for (key, cols) in loci.fields.items(): for col in cols: col_pos = selected_columns.index(col) result_row.append(row[col_pos]) result_row = result_row + ["", loc_a_val, loc_b_val] + unselected_rows output_rows += [[zstr(s) for s in result_row]] if haplotypes.defined: row_val = row[selected_columns.index(haplotypes.column)] if row_val: haplotype_val = haplotypes.lookup[row_val] else: haplotype_val = 0 haplotype_res = [id_field, pop, haplotype_val] haplotype_rows += [[zstr(s) for s in haplotype_res]] # depending on our driver, handle writing dependent on type if format_type == 'Excel': # initialize our spreadsheet workbook = xlwt.Workbook() # codominant data worksheet_co = workbook.add_sheet('Codominant') for (i, row) in enumerate(output_rows): for (j, val) in enumerate(row): worksheet_co.write(i, j, val) # the haplotype data if haplotypes.defined: worksheet_hap = workbook.add_sheet('Haplotype') for (i, row) in enumerate(haplotype_rows): for (j, val) in enumerate(row): worksheet_hap.write(i, j, val) # write out a mapping of haplotype names worksheet_hap_map = workbook.add_sheet('Haplotype Map') worksheet_hap_map.write(0, 0, haplotypes.column) worksheet_hap_map.write(0, 1, 'Numeric Code') for (i, (hap, code)) in enumerate(haplotypes.lookup.items(), start=1): worksheet_hap_map.write(i, 0, hap) worksheet_hap_map.write(i, 1, code) workbook.save(output_name) else: # create a CSV writer writer = csv.writer(output_file, dialect='excel', quotechar='"', quoting=csv.QUOTE_ALL) for row in output_rows: writer.writerow(row) utils.msg("Exported results saved to %s." % output_name) # Close Output text file output_file.close() if mode == 'toolbox': time.sleep(4) arcpy.env.addOutputsToMap = add_output
def run_geodesic_gp(input_fc, unit_factor, output_matrix, row_count, is_spagedi): input_fc_mem = 'in_memory/input_fc' try: utils.msg("Copying features into memory...") arcpy.CopyFeatures_management(input_fc, input_fc_mem) except Exception as e: msg = "Unable to copy features into memory." utils.msg(msg, mtype='error', exception=e) sys.exit() # get the spatial reference of our input, determine the type desc = arcpy.Describe(input_fc_mem) sr = desc.spatialReference if sr.type not in ['Geographic', 'Projected']: msg = "This tools only works with geographic or projected data." utils.msg(msg, mtype='error') sys.exit() utils.msg("Finding all input points...") distance_matrix = OrderedDict() points = OrderedDict() records = arcpy.da.SearchCursor(input_fc_mem, ['OID@', 'SHAPE@XY']) for row in records: (fid, point) = row points[fid] = arcpy.Point(point[0], point[1]) indicator = 0 utils.msg("Computing distances...") for (fid, from_point) in points.items(): pct_progress = int(fid / float(row_count)*100) if pct_progress > indicator: indicator = pct_progress utils.msg("{0}%".format(indicator)) p1 = points[fid] distance_matrix[fid] = OrderedDict() for (to_fid, to_point) in points.items(): if to_fid == fid: dist = 0 elif distance_matrix.has_key(to_fid) and \ distance_matrix[to_fid][fid] is not None: # here, modeling a symmetrical matrix dist = distance_matrix[to_fid][fid] else: p2 = points[to_fid] if p1.equals(p2): dist = 0 else: # Each Polyline initialization must pay the COM object # gods, and ends up making this much more expensive than # AO C++ or even comtypes calls. line = arcpy.Polyline(arcpy.Array([p1, p2]), sr) # distance, always returned in meters, scale by our # expected result units. dist = line.getLength("GEODESIC") * unit_factor distance_matrix[fid][to_fid] = dist utils.msg("Distance matrix calculations complete.") # FIXME: generate it as a CSV file, then do TableToTable to pull it back in # Now compute the lines between these locations. try: # copy the final result back to disk. utils.msg("Writing results to disk...") # The SPAGeDi matrix format are described in section 3.7 of the manual. if is_spagedi: first_header_cell = "M%i" % row_count sep = "\t" else: first_header_cell = "" sep = "," with open(output_matrix, 'w') as csv: # initialize with our header row output_rows = [[first_header_cell] + \ [str(s) for s in distance_matrix.keys()]] for (fid, row) in distance_matrix.items(): res = [str(fid)] + [utils.xstr(s) for s in row.values()] output_rows.append(res) for row in output_rows: csv.write("{0}\n".format(sep.join(row))) if is_spagedi: csv.write("END\n") except Exception as e: utils.msg("Error creating distance matrix.", mtype='error', exception=e) sys.exit()
def index(url,name,type_): from utils import xstr, samealphabetic, hassamealphabetic from reddit import load_subredditsFile, parse_subreddit_entry, create_default_subreddits, assemble_reddit_filter_string, ret_sub_info, ret_settings_type_default_icon ## this is where the main screen is created if not os.path.exists(subredditsFile): #if not os.path.exists(subredditsFile): create_default_subreddits() #if os.path.exists(subredditsPickle): # subreddits_dlist=load_dict(subredditsPickle) #log( pprint.pformat(subreddits_dlist, indent=1) ) #for e in subreddits_dlist: log(e.get('entry_name')) #testing code #h="as asd [S]asdasd[/S] asdas " #log(markdown_to_bbcode(h)) #addDir('test', "url", "next_mode", "", "subreddit" ) #liz = xbmcgui.ListItem(label="test", label2="label2", iconImage="DefaultFolder.png") #u=sys.argv[0]+"?url=&mode=callwebviewer&type=" #xbmcplugin.addDirectoryItem(handle=pluginhandle, url=u, listitem=liz, isFolder=False) #liz = xbmcgui.ListItem().fromString('Hello World') #xbmcplugin.addDirectoryItem(handle=pluginhandle, listitem=liz, isFolder=False) subredditsFile_entries=load_subredditsFile() subredditsFile_entries.sort(key=lambda y: y.lower()) addtl_subr_info={} #this controls what infolabels will be used by the skin. very skin specific. # for estuary, this lets infolabel:plot (and genre) show up below the folder # giving us the opportunity to provide a shortcut_description about the shortcuts xbmcplugin.setContent(pluginhandle, "mixed") #files, songs, artists, albums, movies, tvshows, episodes, musicvideos next_mode='listSubReddit' for subreddit_entry in subredditsFile_entries: #strip out the alias identifier from the subreddit string retrieved from the file so we can process it. #subreddit, alias = subreddit_alias(subreddit_entry) addtl_subr_info=ret_sub_info(subreddit_entry) entry_type, subreddit, alias, shortcut_description=parse_subreddit_entry(subreddit_entry) #log( subreddit + " " + shortcut_description ) #url= urlMain+"/r/"+subreddit+"/.json?"+nsfw+allHosterQuery+"&limit="+itemsPerPage icon=default_icon='' #addon_path+"/resources/skins/Default/media/"+ret_settings_type_default_icon(entry_type) #log(' %s icon=%s' %(subreddit, icon)) url= assemble_reddit_filter_string("",subreddit, "yes") #log("assembled================="+url) if subreddit.lower() in ["all","popular"]: addDir(subreddit, url, next_mode, icon, subreddit, { "plot": translation(30009) } ) #Displays the currently most popular content from all of reddit else: if addtl_subr_info: #if we have additional info about this subreddit #log(repr(addtl_subr_info)) title=xstr(addtl_subr_info.get('title'))+'\n' display_name=xstr(addtl_subr_info.get('display_name')) if samealphabetic( title, display_name): title='' #if re.sub('\W+','', display_name.lower() )==re.sub('\W+','', title.lower()): title='' #display_name=re.sub('\W+','', display_name.lower() ) #title=re.sub('\W+','', title.lower()) header_title=xstr(addtl_subr_info.get('header_title')) public_description=xstr( addtl_subr_info.get('public_description')) if samealphabetic( header_title, public_description): public_description='' if samealphabetic(title,public_description): public_description='' #if hassamealphabetic(header_title,title,public_description): public_description='' if entry_type=='subreddit': display_name='r/'+display_name shortcut_description='[COLOR cadetblue][B]%s[/B][/COLOR]\n%s[I]%s[/I]\n%s' %(display_name,title,header_title,public_description ) icon=addtl_subr_info.get('icon_img') banner=addtl_subr_info.get('banner_img') header=addtl_subr_info.get('header_img') #usually the small icon on upper left side on subreddit screen #log( subreddit + ' icon=' + repr(icon) +' header=' + repr(header)) #picks the first item that is not None icon=next((item for item in [icon,banner,header] if item ), '') or default_icon addDirR(alias, url, next_mode, icon, type_=subreddit, listitem_infolabel={ "plot": shortcut_description }, file_entry=subreddit_entry, banner_image=banner ) else: addDirR(alias, url, next_mode, icon, subreddit, { "plot": shortcut_description }, subreddit_entry ) addDir("[B]- "+translation(30001)+"[/B]", "", 'addSubreddit', "", "", { "plot": translation(30006) } ) #"Customize this list with your favorite subreddit." addDir("[B]- "+translation(30005)+"[/B]", "",'searchReddits', "", "", { "plot": translation(30010) } ) #"Search reddit for a particular post or topic xbmcplugin.endOfDirectory(pluginhandle)
def main(input_features=None, where_clause=None, order_by=None, output_name=None, mode=settings.mode): # get the spatial reference of our input, determine the type desc = arcpy.Describe(input_features) sr = desc.spatialReference if sr.type not in ['Geographic', 'Projected']: utils.msg("This tools only works with geographic or projected data.", mtype='error') sys.exit() # Find our Loci columns. loci = utils.Loci(input_features) utils.msg("loci set: {0}".format(",".join(loci.names))) # sql clause can be prefix or suffix; set up ORDER BY sql_clause = (None, "ORDER BY {0} ASC".format(order_by)) pops = OrderedDict() # query the input_features in ascending order; filtering as needed rows = arcpy.da.SearchCursor(input_features, order_by, where_clause, "", "", sql_clause) row_count = 0 for row in rows: row_count += 1 pop = row[0] if pops.has_key(pop): pops[pop] +=1 else: pops[pop] = 1 # Start with any number of header lines describing what this # file is, each line should be prefixed with //. comments = """// Export to SPAGeDi from the input `{input_features}`. // Export occurred on {datetime} // Coordinates are in {sr_name}, a {sr_type} coordinate system.""".format( input_features=input_features, datetime=datetime.now(), \ sr_name=sr.name, sr_type=sr.type) comment_row = [comments] """ There are three possible ways to specify population (SPAGeDi manual, 2.4): 1) as categorical groups, where one population includes all individuals sharing the same categorical variable. 2) as spatial groups, where a spatial group includes all individuals sharing the same spatial coordinates and following each other in the data file. 3) as spatio-categorical groups, where a spatio-categorical group includes all individuals belonging to both the same spatial group and categorical group. When populations are defined using the categorical variable, the spatial coordinates of a given population are computed by averaging the coordinates of the individuals it contains. """ # FIXME: another spot where 'observation_id' differs from 'individual_id'; # deduplicate in order to have one value PER individual PER population. categories = len(pops.keys()) # FIXME: presumes only two coords, SPAGeDi supports three. Extract depth? if sr.type == 'Geographic': # Assumes decimal degrees; based on the statement 'if the number of # spatial coordinates is set to -2, latitudes and longitudes must be # given in degrees with decimal, using negative numbers for Southern # latitude or Western longitude. xy_type = -2 # geographic data expected to be (lat, lon) loc_a = settings.y_coord loc_b = settings.x_coord else: # two coordinates in a projected space xy_type = 2 loc_a = settings.x_coord loc_b = settings.y_coord num_loci = loci.count # FIXME: loci number of digits, defined in SPAGeDi manual 3.1 as "number # of digits used to code one allele (1 to 3); or set a value =0 (in fact # the value given for missing data) in the case of dominant markers". loci_digits = 2 # get the maximum number of different values per loci max_ploidy = max(map(len, loci.fields.values())) # 1st line: set of 6 numbers separated by a tabulation representing: header_row = [ row_count, # number of invidivuals categories, # number of categories xy_type, # number of coordinates num_loci, # number of loci loci_digits, # number of digits used to code one allele max_ploidy # max ploidy in the data ] # 2nd line: # of distance intervals; upper distance of each interval. # note 1: alternatively you can enter only the desired number of intervals # preceded by a negative sign; the program then defines the # n maximal distances in such a way that the number of pairwise # comparisons within each distance interval is approx. constant. # note 2: if you do not wish distance intervals, put 0 # note 3: if you use latitude + longitude, distance intervals # must be given in km. # TODO: what is this and how do we use it effectively? distances_row = [0] # 3rd line: column labels (<=15 characters). base_cols = [settings.id_field, order_by, loc_a, loc_b] labels_row = base_cols + loci.columns # where_clause is used to ensure only those records with genetic data # are copied to the output. selected_columns = base_cols + loci.columns rows = arcpy.da.SearchCursor(input_features, selected_columns, \ where_clause, "", "", sql_clause) data_rows = [] for row in rows: # our two string fields can't contain spaces, based on Autocio.c: 3857 id_field = str(row[0]).replace(" ", "_") pop_field = str(row[1]).replace(" ", "_") # 'order_by', or population 'group by' loc_a_val = row[2] loc_b_val = row[3] data_row = [id_field, pop_field, loc_a_val, loc_b_val] for (key, cols) in loci.fields.items(): # Loci data can be encoded in a number of formats, the values # separated by any non-numeric values (SPAGeDi manual, 3.2.1). # Here, we use spaces. loci_values = [] for col in cols: col_pos = selected_columns.index(col) allele_val = row[col_pos] # FIXME: handle missing and incomplete genotypes. if allele_val is None: allele_val = "0" else: allele_val = str(allele_val) loci_len = len(allele_val) if loci_len > loci_digits: loci_digits = loci_len loci_values.append(allele_val) data_row.append(" ".join(loci_values)) data_rows.append(data_row) # update header based on revised loci_digits header_row[4] = loci_digits # Now compute the lines between these locations. try: # copy the final result back to disk. utils.msg("Writing results to disk...") output_rows = [comment_row, header_row, distances_row, labels_row] + \ data_rows # SPAGeDi expects tab-delimited outputs sep = "\t" with open(output_name, 'w') as output_file: for raw_row in output_rows: # convert all data to strings row = [utils.xstr(s) for s in raw_row] output_file.write("{0}\n".format(sep.join(row))) # after the last line of invidivual data the word END is required. output_file.write("END\n") except Exception as e: utils.msg("Error creating output file.", mtype='error', exception=e) sys.exit() utils.msg("Exported results saved to %s." % output_name) if mode == 'toolbox': time.sleep(4)
def index(url, name, type_): from utils import xstr, samealphabetic, hassamealphabetic from reddit import load_subredditsFile, parse_subreddit_entry, create_default_subreddits, assemble_reddit_filter_string, ret_sub_info, ret_settings_type_default_icon ## this is where the main screen is created if not os.path.exists( subredditsFile): #if not os.path.exists(subredditsFile): create_default_subreddits() #if os.path.exists(subredditsPickle): # subreddits_dlist=load_dict(subredditsPickle) #log( pprint.pformat(subreddits_dlist, indent=1) ) #for e in subreddits_dlist: log(e.get('entry_name')) #testing code #h="as asd [S]asdasd[/S] asdas " #log(markdown_to_bbcode(h)) #addDir('test', "url", "next_mode", "", "subreddit" ) #liz = xbmcgui.ListItem(label="test", label2="label2", iconImage="DefaultFolder.png") #u=sys.argv[0]+"?url=&mode=callwebviewer&type=" #xbmcplugin.addDirectoryItem(handle=pluginhandle, url=u, listitem=liz, isFolder=False) #liz = xbmcgui.ListItem().fromString('Hello World') #xbmcplugin.addDirectoryItem(handle=pluginhandle, listitem=liz, isFolder=False) subredditsFile_entries = load_subredditsFile() subredditsFile_entries.sort(key=lambda y: y.lower()) addtl_subr_info = {} #this controls what infolabels will be used by the skin. very skin specific. # for estuary, this lets infolabel:plot (and genre) show up below the folder # giving us the opportunity to provide a shortcut_description about the shortcuts xbmcplugin.setContent( pluginhandle, "mixed" ) #files, songs, artists, albums, movies, tvshows, episodes, musicvideos next_mode = 'listSubReddit' for subreddit_entry in subredditsFile_entries: #strip out the alias identifier from the subreddit string retrieved from the file so we can process it. #subreddit, alias = subreddit_alias(subreddit_entry) addtl_subr_info = ret_sub_info(subreddit_entry) entry_type, subreddit, alias, shortcut_description = parse_subreddit_entry( subreddit_entry) #log( subreddit + " " + shortcut_description ) #url= urlMain+"/r/"+subreddit+"/.json?"+nsfw+allHosterQuery+"&limit="+itemsPerPage icon = default_icon = '' #addon_path+"/resources/skins/Default/media/"+ret_settings_type_default_icon(entry_type) #log(' %s icon=%s' %(subreddit, icon)) url = assemble_reddit_filter_string("", subreddit, "yes") #log("assembled================="+url) if subreddit.lower() in ["all", "popular"]: addDir(subreddit, url, next_mode, icon, subreddit, { "plot": translation(30009) }) #Displays the currently most popular content from all of reddit else: if addtl_subr_info: #if we have additional info about this subreddit #log(repr(addtl_subr_info)) title = xstr(addtl_subr_info.get('title')) + '\n' display_name = xstr(addtl_subr_info.get('display_name')) if samealphabetic(title, display_name): title = '' #if re.sub('\W+','', display_name.lower() )==re.sub('\W+','', title.lower()): title='' #display_name=re.sub('\W+','', display_name.lower() ) #title=re.sub('\W+','', title.lower()) header_title = xstr(addtl_subr_info.get('header_title')) public_description = xstr( addtl_subr_info.get('public_description')) if samealphabetic(header_title, public_description): public_description = '' if samealphabetic(title, public_description): public_description = '' #if hassamealphabetic(header_title,title,public_description): public_description='' if entry_type == 'subreddit': display_name = 'r/' + display_name shortcut_description = '[COLOR cadetblue][B]%s[/B][/COLOR]\n%s[I]%s[/I]\n%s' % ( display_name, title, header_title, public_description) icon = addtl_subr_info.get('icon_img') banner = addtl_subr_info.get('banner_img') header = addtl_subr_info.get( 'header_img' ) #usually the small icon on upper left side on subreddit screen #log( subreddit + ' icon=' + repr(icon) +' header=' + repr(header)) #picks the first item that is not None icon = next((item for item in [icon, banner, header] if item), '') or default_icon addDirR(alias, url, next_mode, icon, type_=subreddit, listitem_infolabel={"plot": shortcut_description}, file_entry=subreddit_entry, banner_image=banner) else: addDirR(alias, url, next_mode, icon, subreddit, {"plot": shortcut_description}, subreddit_entry) addDir("[B]- " + translation(30001) + "[/B]", "", 'addSubreddit', "", "", {"plot": translation(30006) }) #"Customize this list with your favorite subreddit." addDir("[B]- " + translation(30005) + "[/B]", "", 'searchReddits', "", "", {"plot": translation(30010) }) #"Search reddit for a particular post or topic xbmcplugin.endOfDirectory(pluginhandle)
print "START TIME: [" + time.strftime("%Y-%m-%d-%H:%M:%S", time.localtime()) + "]" print "START COMMAND: " + COMMAND outputpath, outputname, user, password = export_hive_to_file(COMMAND, utils.listToString(parametersAliasList), type, outputname, ftpuser) try: email("Report: Data Export Utility","complete!","smithmicro.com",emailaddr,"{'text':'email from export report', 'html':'<p>Your file is now complete. To access your report, click on this link: <a href=\"ftp://"+os.uname()[1]+"/"+type+"/"+outputname+"\">"+outputname+"</a> and enter your username and password below.</p><p>Username: <b>"+user.strip()+"</b><br />Password: <b>"+password.strip()+"</b></p>'}") except: print "The Email Server has Raised an Error" if bool(scp) == True: email_addr = user.strip() scp_method(email_addr, outputname, target_user, target_server, target_dir) except IOError, e: print "I/O error(%(error0)s):(%(error1)s)" % { "error0":e[0],"error1":e[1] } email("Export_Report","failed","analytics.smithmicro.com",emailaddr,"\nError:\nI/O error("+utils.xstr(e[0])+"): "+xstr(e[1])+"\n"+utils.xstr(sys.exc_info()[0])) except NameError, e: print "An internal error has occured, possibly user input or system is temporarily unavailable." email("Export_Report","failed","analytics.smithmicro.com",emailaddr,"\nError:\nNameError("+utils.xstr(sys.exc_info()[0])+utils.xstr(e[0])+"): "+"An internal error has occured, possibly user input or system is temporarily unavailable. Please contact your administrator or SmithMicro at [email protected].") raise except: print "Unexpected Error: " + utils.xstr(sys.exc_info()[0]) email("Export_Report","failed","analytics.smithmicro.com",emailaddr,"\nError:\n"+utils.xstr(sys.exc_info()[0])) print "\n+++++===HIVE END COMMAND===+++++++: " + COMMAND + "\n" return COMMAND def main(argv=None): if argv is None: argv = sys.argv[1:] else: argv = argv.split() try: #print 'ARGV :', argv[:] try: