def show_action_edit(self):
now = Date()
sdf = SimpleDateFormat("yyyy")
year = int(sdf.format(now))
sdf = SimpleDateFormat("M")
month = int(sdf.format(now))
sdf = SimpleDateFormat("d")
day = int(sdf.format(now))
request.setAttribute("default_year", year)
request.setAttribute("default_month", month)
request.setAttribute("default_day", day)
return "/WEB-INF/ftl/course/show_preparecourse_action_create.ftl"
def __newDoc(self):
self.oid = self.object.getId()
self.pid = self.payload.getId()
metadataPid = self.params.getProperty("metaPid", "DC")
self.utils.add(self.index, "storage_id", self.oid)
if self.pid == metadataPid:
self.itemType = "object"
else:
self.oid += "/" + self.pid
self.itemType = "datastream"
self.utils.add(self.index, "identifier", self.pid)
self.utils.add(self.index, "id", self.oid)
self.utils.add(self.index, "item_type", self.itemType)
self.utils.add(self.index, "last_modified", self.last_modified)
self.utils.add(self.index, "harvest_config", self.params.getProperty("jsonConfigOid"))
self.utils.add(self.index, "harvest_rules", self.params.getProperty("rulesOid"))
self.item_security = []
self.owner = self.params.getProperty("owner", "guest")
formatter = SimpleDateFormat('yyyyMMddHHmmss')
self.params.setProperty("last_modified", formatter.format(Date()))
self.utils.add(self.index, "date_object_created", self.params.getProperty("date_object_created"))
self.params.setProperty("date_object_modified", time.strftime("%Y-%m-%dT%H:%M:%SZ", time.localtime()) )
self.utils.add(self.index, "date_object_modified", self.params.getProperty("date_object_modified"))
def change_view(self):
self.vlayout.removeAllViews()
self.vlayout.addView(self.text_result)
calendar = Calendar.getInstance()
dateformat = SimpleDateFormat('yyyy / MM / dd')
now = dateformat.format(calendar.getTime())
self.text_result.setText(now)
def __newDoc(self):
self.oid = self.object.getId()
self.pid = self.payload.getId()
metadataPid = self.params.getProperty("metaPid", "DC")
self.utils.add(self.index, "storage_id", self.oid)
if self.pid == metadataPid:
self.itemType = "object"
else:
self.oid += "/" + self.pid
self.itemType = "datastream"
self.utils.add(self.index, "identifier", self.pid)
self.utils.add(self.index, "id", self.oid)
self.utils.add(self.index, "item_type", self.itemType)
self.utils.add(self.index, "last_modified", self.last_modified)
self.utils.add(self.index, "harvest_config",
self.params.getProperty("jsonConfigOid"))
self.utils.add(self.index, "harvest_rules",
self.params.getProperty("rulesOid"))
self.item_security = []
self.owner = self.params.getProperty("owner", "guest")
formatter = SimpleDateFormat('yyyyMMddHHmmss')
self.params.setProperty("last_modified", formatter.format(Date()))
self.utils.add(self.index, "date_object_created",
self.params.getProperty("date_object_created"))
self.params.setProperty(
"date_object_modified",
time.strftime("%Y-%m-%dT%H:%M:%SZ", time.localtime()))
self.utils.add(self.index, "date_object_modified",
self.params.getProperty("date_object_modified"))
def process_record(self):
content = {}
self.set_from_task_vars('u_request', content)
self.set_from_task_vars('u_application_name', content)
self.set_from_task_vars('cmdb_ci', content)
self.set_from_task_vars('priority', content)
self.set_from_task_vars('state', content)
self.set_from_task_vars('assignment_group', content)
self.set_from_task_vars('assigned_to', content)
self.set_from_task_vars('due_date', content)
self.set_from_task_vars('short_description', content)
self.set_from_task_vars('description', content)
# Dates need to be converted
sdf = SimpleDateFormat("MM-dd-yyyy HH:mm:ss")
content['due_date'] = sdf.format(self.task_vars['due_date'])
# Also sending release info.
content['x_xlbv_xl_release_identifier'] = str(release.id)
content['x_xlbv_xl_release_state'] = str(release.status)
logger.debug('process_record : send create_record request...')
logger.debug('process_record : table name: ' + self.table_name)
logger.debug('process_record : content... ')
logger.debug(content)
response = self.sn_client.create_record(self.table_name, content,
getCurrentTask().getId())
logger.debug('process_record : response...')
logger.debug(response)
return response
def flowFileDates(hmsRunName):
#will not be accurate if there is more than one set of data--specify start date in config file?
"""Find the dates of any FLOW files in the DSS catalog"""
from java.text import SimpleDateFormat
dateFormat = SimpleDateFormat("ddMMMyyyy")
print("Getting dates from " + hmsRunName + "...")
dates = []
#print(dss.getCatalogedPathnames())
flowFiles = filter(
lambda f:
((f.split('/')[3] == 'FLOW') and (f.split('/')[6] ==
('RUN:' + hmsRunName.upper()))),
dss.getCatalogedPathnames())
#print(flowFiles)
candidateDates = map(lambda x: x.split('/')[4], flowFiles)
#print(candidateDates)
for d in candidateDates:
if d[0:2].isdigit() and d[2:5].isalpha() and d[5:9].isdigit():
date = dateFormat.parse(d)
dateAlreadyFound = any(lambda x: x.equals(date), dates)
if not dateAlreadyFound:
dates.append(date)
dates.sort(lambda a, b: a.compareTo(b))
return map(lambda d: dateFormat.format(d).upper(), dates)
def __init__(self, selectFields):
BaseQuery.__init__(self, selectFields)
self.params = ParamUtil(request)
self.prepareCourseId = None
self.createUserId = None
self.k = self.params.getStringParam("k") #查询关键字
self.ktype = self.params.getStringParam("ktype") #查询类型[关键字对应的类型]
if self.ktype == None:
self.ktype = "1" #默认关键字查询标题
self.unit = self.params.getStringParam("unit") #主备人所属机构
self.course_BeginDate = self.params.getStringParam("course_BeginDate")
self.course_EndDate = self.params.getStringParam("course_EndDate")
self.subjectId = self.params.getIntParamZeroAsNull("subjectId")
self.gradeId = self.params.getIntParamZeroAsNull("gradeId")
request.setAttribute("subjectId", self.subjectId)
request.setAttribute("gradeId", self.gradeId)
request.setAttribute("k", self.k)
request.setAttribute("ktype", self.ktype)
request.setAttribute("unit", self.unit)
request.setAttribute("course_BeginDate", self.course_BeginDate)
request.setAttribute("course_EndDate", self.course_EndDate)
self.orderType = 0
self.status = None
self.stage = None #集备执行的阶段，正在进行running；已经完成finishaed；还未进行will ;recommend 推荐的
self.containChild = None #准确学科的查询
self.prepareCoursePlanId = None
self.prepareCourseGenerated = True
self.custormAndWhere = None #自定义条件查询
sft = SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
self.nowDate = sft.format(Date())
def create_sale(self, sale):
values = ContentValues()
values.put("person", sale['person'].lower())
values.put("total", sale['value'])
values.put("description", sale['description'].replace('\n',
' ').lower())
values.put("paid", sale['paid'])
calendar = Calendar.getInstance()
dateformat = SimpleDateFormat('yyyy/MM/dd HH:mm')
now = dateformat.format(calendar.getTime())
values.put("date", now)
db = self.getWritableDatabase()
db.insertWithOnConflict("sale", None, values,
SQLiteDatabase.CONFLICT_REPLACE)
#remove quantity from products
products = sale['description'].split('\n')
for product in products:
name, quantity = product.split(':')
quantity = int(quantity)
db.execSQL(
"UPDATE product SET quantity = quantity - %d WHERE name='%s'" %
(quantity, name))
db.close()
def getVigentStretchesQuery(store, fecha):
#query = getCarreterasManager().getVigentStretchesQuery(store, fecha)
dateFormatter = SimpleDateFormat("dd/MM/yyyy")
formatedDate = dateFormatter.format(fecha)
filtro = "( fecha_entrada <= '%s' OR fecha_entrada IS NULL) AND ('%s' <= fecha_salida OR fecha_salida IS NULL)" % (
formatedDate, formatedDate)
query = store.createFeatureQuery()
query.addFilter(filtro)
return query
def getDate(self):
calendar = GregorianCalendar()
date = calendar.getTime()
dayFormat = SimpleDateFormat("dd")
monthFormat = SimpleDateFormat("MM")
yearFormat = SimpleDateFormat("yyyy")
DAY = int(dayFormat.format(date))
MONTH = int(monthFormat.format(date))
YEAR = int(yearFormat.format(date))
if MONTH < 10:
TEMP1 = "%d0%d" % (YEAR, MONTH)
else:
TEMP1 = "%d%d" % (YEAR, MONTH)
if DAY < 10:
CURRENTDATE = "%d0%d" % (TEMP1, DAY)
else:
CURRENTDATE = "%d%d" % (TEMP1, DAY)
return CURRENTDATE
def getDate(self):
calendar = GregorianCalendar()
date = calendar.getTime()
dayFormat = SimpleDateFormat("dd")
monthFormat = SimpleDateFormat("MM")
yearFormat = SimpleDateFormat("yyyy")
DAY = int(dayFormat.format(date))
MONTH = int(monthFormat.format(date))
YEAR = int(yearFormat.format(date))
if MONTH < 10:
TEMP1 = "%d0%d" % (YEAR, MONTH)
else:
TEMP1 = "%d%d" % (YEAR, MONTH)
if DAY < 10:
CURRENTDATE = "%d0%d" % (TEMP1, DAY)
else:
CURRENTDATE = "%d%d" % (TEMP1, DAY)
return CURRENTDATE
def __init__(self, selectFields):
BaseQuery.__init__(self, selectFields)
self.params = ParamUtil(request)
self.prepareCourseId = None
self.userId = None
self.privateContentExist = None #是否只显示有内容的个案 默认None,全部显示 True,查询有内容的,False 查询无内容的
self.stage = None #集备执行的阶段，正在进行running；已经完成finishaed；还未进行will
sft = SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
self.nowDate = sft.format(Date())
def getSqlDateInGMT(date):
"""
@type: long->str
@param: date - timestamp
"""
dateObj = Date(date)
df = SimpleDateFormat()
df.setTimeZone(TimeZone.getTimeZone("GMT"))
# Default Java Date.toString() format
df.applyPattern('EEE MMM dd HH:mm:ss zzz yyyy')
return df.format(dateObj)
def setupjob(job, args):
"""
Set up a job to run on a date range of directories.
Jobs expect two arguments, startdate and enddate, both in yyyy-MM-dd format.
"""
import java.text.SimpleDateFormat as SimpleDateFormat
import java.util.Date as Date
import java.util.Calendar as Calendar
import com.mozilla.util.DateUtil as DateUtil
import com.mozilla.util.DateIterator as DateIterator
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat as FileInputFormat
import org.apache.hadoop.mapreduce.lib.input.SequenceFileAsTextInputFormat as MyInputFormat
if len(args) != 3:
raise Exception(
"Usage: <testpilot_study> <startdate-YYYY-MM-DD> <enddate-YYYY-MM-DD>"
)
# use to collect up each date in the given range
class MyDateIterator(DateIterator):
def __init__(self):
self._list = []
def get(self):
return self._list
def see(self, aTime):
self._list.append(aTime)
sdf = SimpleDateFormat(dateformat)
study = args[0]
startdate = Calendar.getInstance()
startdate.setTime(sdf.parse(args[1]))
enddate = Calendar.getInstance()
enddate.setTime(sdf.parse(args[2]))
dates = MyDateIterator()
DateUtil.iterateByDay(startdate.getTimeInMillis(),
enddate.getTimeInMillis(), dates)
paths = []
for d in dates.get():
paths.append(pathformat % (study, sdf.format(Date(d))))
job.setInputFormatClass(MyInputFormat)
FileInputFormat.setInputPaths(job, ",".join(paths))
job.getConfiguration().set("org.mozilla.jydoop.mappertype", "TEXT")
def __init__(self, selectFields):
BaseQuery.__init__(self, selectFields)
self.params = ParamUtil(request)
self.orderType = 0
self.createUserId = None
self.ownerType = None
self.ownerId = None
self.status = None
self.qryDate = None
self.k = self.params.getStringParam("k")
self.filter = self.params.getStringParam("filter")
sft = SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
self.nowDate = sft.format(Date())
def actionPerformed(self,actionEvent):
self.scl_long_tuneup_controller.getMessageTextField().setText("")
rightNow = Calendar.getInstance()
date_format = SimpleDateFormat("MM.dd.yyyy")
time_str = date_format.format(rightNow.getTime())
fc = JFileChooser(constants_lib.const_path_dict["XAL_XML_ACC_FILES_DIRS_PATH"])
fc.setDialogTitle("Save SCL data into the SCL_new.xdxf file")
fc.setApproveButtonText("Save")
fl_filter = FileNameExtensionFilter("SCL Acc File",["xdxf",])
fc.setFileFilter(fl_filter)
fc.setSelectedFile(File("SCL_"+time_str+".xdxf"))
returnVal = fc.showOpenDialog(self.scl_long_tuneup_controller.linac_wizard_document.linac_wizard_window.frame)
if(returnVal == JFileChooser.APPROVE_OPTION):
fl_out = fc.getSelectedFile()
fl_path = fl_out.getPath()
if(fl_path.rfind(".xdxf") != (len(fl_path) - 5)):
fl_out = File(fl_out.getPath()+".xdxf")
#---------prepare the XmlDataAdaptor
root_DA = XmlDataAdaptor.newEmptyDocumentAdaptor()
scl_DA = root_DA.createChild("xdxf")
scl_DA.setValue("date",time_str)
scl_DA.setValue("system","sns")
scl_DA.setValue("version","2.0")
#---- SCLMed
seq_name_arr = ["SCLMed","SCLHigh","HEBT1"]
for seq_name in seq_name_arr:
accl = self.scl_long_tuneup_controller.linac_wizard_document.accl
seq = accl.findSequence(seq_name)
cavs = seq.getAllNodesWithQualifier(AndTypeQualifier().and((OrTypeQualifier()).or(SCLCavity.s_strType)))
quads = seq.getAllNodesWithQualifier(AndTypeQualifier().and((OrTypeQualifier()).or(Quadrupole.s_strType)))
scl_seq_DA = scl_DA.createChild("sequence")
scl_seq_DA.setValue("id",seq.getId())
for quad in quads:
node_DA = scl_seq_DA.createChild("node")
node_DA.setValue("id",quad.getId())
attr_DA = node_DA.createChild("attributes")
field_DA = attr_DA.createChild("magnet")
scl_quad_fields_dict_holder = self.scl_long_tuneup_controller.scl_long_tuneup_init_controller.scl_quad_fields_dict_holder
field_DA.setValue("dfltMagFld",str(scl_quad_fields_dict_holder.quad_field_dict[quad]))
for cav in cavs:
node_DA = scl_seq_DA.createChild("sequence")
node_DA.setValue("id",cav.getId())
attr_DA = node_DA.createChild("attributes")
rf_cav_DA = attr_DA.createChild("rfcavity")
cav_wrappper = self.scl_long_tuneup_controller.getCav_WrapperForCavId(cav.getId())
(amp,phase) = (cav_wrappper.designAmp,cav_wrappper.designPhase)
rf_cav_DA.setValue("amp",float("%8.5f"%amp))
rf_cav_DA.setValue("phase",float("%8.3f"%phase))
root_DA.writeTo(fl_out)
def execute(self):
self.parentGuid=self.params.safeGetStringParam("parentGuid")
self.parentType=self.params.safeGetStringParam("parentType")
self.year = self.params.safeGetIntParam("year")
self.month = self.params.safeGetIntParam("month")
self.day = self.params.safeGetIntParam("day")
if self.year==0:
now=Date()
sdf = SimpleDateFormat("yyyy")
self.year= sdf.format(now)
sdf = SimpleDateFormat("M")
self.month= sdf.format(now)
sdf = SimpleDateFormat("d")
self.day= sdf.format(now)
self.calendars = self.calendar_svc.getCalendars(self.parentGuid,self.parentType,int(self.year),int(self.month))
request.setAttribute("parentGuid",self.parentGuid)
request.setAttribute("parentType",self.parentType)
request.setAttribute("calendars",self.calendars)
request.setAttribute("year",self.year)
request.setAttribute("month",self.month)
request.setAttribute("day",self.day)
return "/WEB-INF/mod/calendarevent/GetCalendarMsg.ftl"
def dateformat(t, format, language=None):
"""
Format python date to string using Java SimpleDateFormat.
:param t: Python date.
:returns: Format string of the date
"""
jt = jdate(t)
if language is None:
df = SimpleDateFormat(format)
else:
locale = Locale(language)
df = SimpleDateFormat(format, locale)
return df.format(jt)
def __init__(self, imp):
'''Get the metadata from the given dm3 image.
'''
extractor = GatanMetadataExtractor(imp)
self.exposure = extractor.getExposure()
self.magnification = extractor.getMagnification()
self.mag_factor = extractor.getActualMagnification() / self.magnification
self.mag_unit = 'x'
if not Double.isNaN(extractor.getEnergyloss()):
self.energyloss = extractor.getEnergyloss()
else:
self.energyloss = 0
self.date = extractor.getDateAndTime()
date_formater = SimpleDateFormat('yyyyMMdd')
self.date_string = date_formater.format(self.date)
self.name = extractor.getName()
self.prop_dict = {}
def setupjob(job, args):
"""
Set up a job to run on a date range of directories.
Jobs expect two arguments, startdate and enddate, both in yyyy-MM-dd format.
"""
import java.text.SimpleDateFormat as SimpleDateFormat
import java.util.Date as Date
import java.util.Calendar as Calendar
import com.mozilla.util.DateUtil as DateUtil
import com.mozilla.util.DateIterator as DateIterator
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat as FileInputFormat
import org.apache.hadoop.mapreduce.lib.input.SequenceFileAsTextInputFormat as MyInputFormat
if len(args) != 3:
raise Exception("Usage: <testpilot_study> <startdate-YYYY-MM-DD> <enddate-YYYY-MM-DD>")
# use to collect up each date in the given range
class MyDateIterator(DateIterator):
def __init__(self):
self._list = []
def get(self):
return self._list
def see(self, aTime):
self._list.append(aTime)
sdf = SimpleDateFormat(dateformat)
study = args[0]
startdate = Calendar.getInstance()
startdate.setTime(sdf.parse(args[1]))
enddate = Calendar.getInstance()
enddate.setTime(sdf.parse(args[2]))
dates = MyDateIterator()
DateUtil.iterateByDay(startdate.getTimeInMillis(), enddate.getTimeInMillis(), dates)
paths = []
for d in dates.get():
paths.append(pathformat % (study, sdf.format(Date(d))))
job.setInputFormatClass(MyInputFormat)
FileInputFormat.setInputPaths(job, ",".join(paths));
job.getConfiguration().set("org.mozilla.jydoop.mappertype", "TEXT")
def createFile(self):
if (os.path.exists(self.datafilename) == 0):
fid = open(self.datafilename, 'w')
df = SimpleDateFormat('hh.mm.dd.MM.yyyy')
today = df.format(Date())
print "Writing data to file:" + self.datafilename
print "Writing mca file to:" + self.mcadir
# write datetime
line = " I18_EXAFS_RUN=" + str(self.fileno) + " " + today
print >> fid, line
print >> fid, self.title
print >> fid, self.condition1
print >> fid, self.condition2
print >> fid, self.condition3
print >> fid, 'Sample X=', MicroFocusSampleX.getPosition(
), 'Sample Y=', MicroFocusSampleY.getPosition()
print >> fid, 'comboDCM energy time I0 It drain flu1 flu2 flu3 flu4 flu5 flu6 flu7 flu8 flu9 flutot'
fid.close()
def savePreviousArguments(managedServerName):
from java.io import File
from java.io import FileOutputStream
from java.util import Properties
from java.util import Date
from java.text import SimpleDateFormat
import string
startToEdit()
# parameter on the wsdl ant task call
fileLocation = sys.argv[1].replace("\\", "/")
print "The backup file location is"
print fileLocation
try:
dateFormat = SimpleDateFormat('_d_MMM_yyyy_HH_mm_ss')
date = Date()
formattedDate = dateFormat.format(date)
print formattedDate
except:
print "The date cannot be created/formatted"
try:
propsFile = File(fileLocation + managedServerName + formattedDate +
"_config.bkp")
print propsFile.exists()
if (propsFile.exists() == 0):
propsFile.createNewFile()
except:
print "The file cannot be created on:"
print propsFile.getAbsoluteFile()
dumpStack()
previousProperties = Properties()
print '===> Saving the previous arguments - ' + managedServerName
cd('/Servers/' + managedServerName)
print "Getting the Classpath"
classPath = cmo.getServerStart().getClassPath()
print classPath
if classPath == None:
classPath = ""
previousProperties.setProperty("classPath", classPath)
print "Saving Arguments to file"
previousProperties.store(FileOutputStream(propsFile), None)
print '===> Saved arguments! Please verify the file on:' + fileLocation + "in" + managedServerName
def savePreviousArguments(managedServerName):
from java.io import File
from java.io import FileOutputStream
from java.util import Properties
from java.util import Date
from java.text import SimpleDateFormat
import string
startToEdit()
# parameter on the wsdl ant task call
fileLocation = sys.argv[1].replace("\\","/")
print "The backup file location is"
print fileLocation
try:
dateFormat = SimpleDateFormat('_d_MMM_yyyy_HH_mm_ss')
date = Date()
formattedDate = dateFormat.format(date)
print formattedDate
except:
print "The date cannot be created/formatted"
try:
propsFile = File(fileLocation+ managedServerName + formattedDate+"_config.bkp");
print propsFile.exists()
if(propsFile.exists() == 0):
propsFile.createNewFile()
except:
print "The file cannot be created on:"
print propsFile.getAbsoluteFile()
dumpStack()
previousProperties = Properties()
print '===> Saving the previous arguments - ' + managedServerName
cd('/Servers/'+managedServerName)
print "Getting the VMArgs"
vmArgs = cmo.getServerStart().getArguments()
print vmArgs
if vmArgs == None:
vmArgs = ""
previousProperties.setProperty("vmArgs", vmArgs)
print "Saving Arguments to file"
previousProperties.store(FileOutputStream(propsFile),None)
print '===> Saved arguments! Please verify the file on:'+ fileLocation + "in" + managedServerName
def makeTimeSeriesContainer(station, interval, tz, records, decodeInfo):
global timezones
sdf = SimpleDateFormat("ddMMMyyyy, HH:mm")
if dssTimezone:
if not timezones["DSS"]:
timezones["DSS"] = TimeZone.getTimeZone(
tzInfo[dssTimezone]["JAVA"])
sdf.setTimeZone(timezones["DSS"])
else:
sdf.setTimeZone(timezones["USGS"])
dd, decodeInfo = decodeInfo
cal = Calendar.getInstance()
t = HecTime()
tsc = TimeSeriesContainer()
tsc.interval = interval
times = []
values = []
tsc.quality = None
factor = decodeInfo["DSS_FACTOR"]
for j in range(len(records)):
millis, value = records[j]
cal.setTimeInMillis(millis)
t.set(sdf.format(cal.getTime()))
times.append(t.value())
try:
values.append(float(value) * factor)
except:
values.append(Constants.UNDEFINED)
tsc.times = times
tsc.values = values
tsc.startTime = times[0]
tsc.endTime = times[-1]
tsc.numberValues = len(values)
tsc.timeZoneID = sdf.getTimeZone().getID()
tsc.timeZoneRawOffset = sdf.getTimeZone().getRawOffset()
return tsc
def flowFileDates(hmsRunName):
#will not be accurate if there is more than one set of data--specify start date in config file?
"""Find the dates of any FLOW files in the DSS catalog"""
from java.text import SimpleDateFormat
dateFormat = SimpleDateFormat("ddMMMyyyy")
print("Getting dates from " + hmsRunName + "...")
dates = []
#print(dss.getCatalogedPathnames())
flowFiles = filter(lambda f:((f.split('/')[3] == 'FLOW') and (f.split('/')[6] == ('RUN:'+hmsRunName.upper()))),
dss.getCatalogedPathnames())
#print(flowFiles)
candidateDates = map(lambda x:x.split('/')[4], flowFiles)
#print(candidateDates)
for d in candidateDates:
if d[0:2].isdigit() and d[2:5].isalpha() and d[5:9].isdigit():
date = dateFormat.parse(d)
dateAlreadyFound = any(lambda x:x.equals(date), dates)
if not dateAlreadyFound:
dates.append(date)
dates.sort(lambda a,b:a.compareTo(b))
return map(lambda d: dateFormat.format(d).upper(), dates)
def formatDate(date, sfmt="yyyy-MM-dd'T'HH:mm:ss", tfmt="dd/MM/yyyy"):
dfSource = SimpleDateFormat(sfmt)
dfTarget = SimpleDateFormat(tfmt)
return dfTarget.format(dfSource.parse(date))
formatter.setTimeZone(TimeZone.getTimeZone(snTimeZone))
startDate = snData[startField]
print "------"
try:
print "Schedule Task=> date = %s" % (startDate)
date = formatter.parse(startDate)
print "Schedule Task=> date = %s" % (date)
release = getCurrentRelease()
releaseID = release.id
phaseTitle=targetPhase
taskTitle=targetTask
print "Schedule Task=> Phase / Task = %s / %s" % ( phaseTitle, taskTitle )
phase = phaseApi.searchPhasesByTitle( phaseTitle, releaseID )
print "Schedule Task=> phase = %s" % ( phase )
phaseID = phase[0].id
task = taskApi.searchTasksByTitle( taskTitle, phaseTitle, releaseID)
print "Schedule Task=> task = %s" % ( task )
taskID = task[0].id
myTask = taskApi.getTask( taskID )
myTask.waitForScheduledStartDate = True
myTask.scheduledStartDate = xlrFormat.format(date)
print "Schedule Task=> Task = %s" % ( myTask )
print "Schedule Task=> Task.scheduledStartDate = %s" % ( myTask.scheduledStartDate )
taskApi.updateTask( myTask )
except ParseException :
print "exception
# End try
print "------"
def hdfs_setupjob(job, args):
"""
Similar to the above, but run telemetry data that's already been exported
to HDFS.
Jobs expect two arguments, startdate and enddate, both in yyyyMMdd format.
"""
import java.text.SimpleDateFormat as SimpleDateFormat
import java.util.Date as Date
import java.util.Calendar as Calendar
import java.util.concurrent.TimeUnit as TimeUnit
import com.mozilla.util.DateUtil as DateUtil
import com.mozilla.util.DateIterator as DateIterator
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat as FileInputFormat
import org.apache.hadoop.mapreduce.lib.input.SequenceFileAsTextInputFormat as MyInputFormat
if len(args) != 2:
raise Exception("Usage: <startdate-YYYYMMDD> <enddate-YYYYMMDD>")
# use to collect up each date in the given range
class MyDateIterator(DateIterator):
def __init__(self):
self._list = []
def get(self):
return self._list
def see(self, aTime):
self._list.append(aTime)
sdf = SimpleDateFormat(dateformat)
sdf_hdfs = SimpleDateFormat(hdfs_dateformat)
startdate = Calendar.getInstance()
startdate.setTime(sdf.parse(args[0]))
enddate = Calendar.getInstance()
enddate.setTime(sdf.parse(args[1]))
nowdate = Calendar.getInstance()
# HDFS only contains the last 2 weeks of data (up to yesterday)
startMillis = startdate.getTimeInMillis()
endMillis = enddate.getTimeInMillis()
nowMillis = nowdate.getTimeInMillis()
startDiff = nowMillis - startMillis
if TimeUnit.DAYS.convert(startDiff, TimeUnit.MILLISECONDS) > 14:
raise Exception("HDFS Data only includes the past 14 days of history. Try again with more recent dates or use the HBase data directly.")
endDiff = nowMillis - endMillis
if TimeUnit.DAYS.convert(endDiff, TimeUnit.MILLISECONDS) < 1:
raise Exception("HDFS Data only includes data up to yesterday. For (partial) data for today, use the HBase data directly.")
dates = MyDateIterator()
DateUtil.iterateByDay(startMillis, endMillis, dates)
paths = []
for d in dates.get():
paths.append(hdfs_pathformat % (sdf_hdfs.format(Date(d))))
job.setInputFormatClass(MyInputFormat)
FileInputFormat.setInputPaths(job, ",".join(paths));
def evaluate(self, times, max_time, origins_csv, destinations_csv, csv_writer, split=500, do_merge=False):
'''
evaluate the shortest paths between origins and destinations
uses the routing options set in setup() (run it first!)
Parameters
----------
times: list of date times, the desired start/arrival times for evaluation
origins_csv: file with origin points
destinations_csv: file with destination points
csv_writer: CSVWriter, configured writer to write results
do_merge: merge the results over time, only keeping the best connections
max_time: maximum travel-time in seconds (the smaller this value, the smaller the shortest path tree, that has to be created; saves processing time)
'''
origins = self.otp.loadCSVPopulation(origins_csv, LATITUDE_COLUMN, LONGITUDE_COLUMN)
destinations = self.otp.loadCSVPopulation(destinations_csv, LATITUDE_COLUMN, LONGITUDE_COLUMN)
sources = origins if not self.arrive_by else destinations
n_slices = (sources.size() / split) + 1
if n_slices > 1:
print 'Splitting sources into {} part(s) with {} points each part'.format(n_slices, split)
from_index = 0;
to_index = 0;
i = 1
while True:
if to_index >= sources.size():
break
from_index = to_index
to_index += split
if to_index >= sources.size():
to_index = sources.size()
sliced_sources = sources.get_slice(from_index, to_index)
if n_slices > 1:
print('calculating part {}/{}'.format(i, n_slices))
i += 1
if not self.arrive_by:
origins = sliced_sources
else:
destinations = sliced_sources
self.request.setOrigins(origins)
self.request.setDestinations(destinations)
self.request.setLogProgress(self.print_every_n_lines)
if self.arrive_by:
time_note = ' arrival time '
else:
time_note = 'start time '
# # if evaluation is performed in a time window, routes exceeding the window will be ignored
# # (worstTime already takes care of this, but the time needed to reach the snapped the OSM point is also taken into account here)
# if len(times) > 1:
# print 'Cutoff set: routes with {}s exceeding the time window ({}) will be ignored (incl. time to reach OSM-net)'.format(time_note, times[-1])
# cutoff = times[-1]
# self.request.setCutoffTime(cutoff.year, cutoff.month, cutoff.day, cutoff.hour, cutoff.minute, cutoff.second)
# iterate all times
results = [] # dimension (if not merged): times x targets (origins resp. destinations)
sdf = SimpleDateFormat('HH:mm:ss')
sdf.setTimeZone(TimeZone.getTimeZone("GMT +2"))
for t, date_time in enumerate(times):
# compare seconds since epoch (different ways to get it from java/python date)
epoch = datetime.utcfromtimestamp(0)
time_since_epoch = (date_time - epoch).total_seconds()
self.request.setDateTime(date_time.year, date_time.month, date_time.day, date_time.hour, date_time.minute, date_time.second)
# has to be set every time after setting datetime (and also AFTER setting arriveby)
self.request.setMaxTimeSec(max_time)
msg = 'Starting evaluation of routes with ' + time_note + date_time.strftime(DATETIME_FORMAT)
print msg
results_dt = self.batch_processor.evaluate(self.request)
# if there already was a calculation: merge it with new results
if do_merge and len(results) > 0:
for i, prev_result in enumerate(results[0]):
if prev_result is not None:
prev_result.merge(results_dt[i])
#write and append if no merging is needed (saves memory)
else:
search_time = sdf.format(date_time)
csv_writer.write(results_dt, additional_columns={'search_time': search_time}, append=True)
for r in results_dt:
del(r)
if do_merge:
# flatten the results
results = [r for res in results for r in res]
csv_writer.write(results, append=False)
# F:servletSessionsModule=
# F:systemModule=
# F:threadPoolModule=
# F:transactionModule=
# F:webAppModule=F'
#-----------------------------------------------------------------
if (len(sys.argv) == 2):
# Diretorio para escrita das metricas coletadas
global OUTPUT_PATH
global ERROR_FILE
modules = sys.argv[0]
OUTPUT_PATH = sys.argv[1]
ERROR_FILE = os.path.join(OUTPUT_PATH, DAY_FORMAT.format(Date()) + "_errorLog.log")
splitModules = modules.split(',')
for aModule in splitModules:
# Remove eventuais espacos em branco existentes
aModule = String(aModule).replace(String(" "),String(""))
processServerMetrics(aModule)
else:
print "Este script requer dois parametros para sua execucao:"
print "1 - Lista de modulos para os quais serao extraidas as metricas."
print "Segue a lista de modules disponiveis:"
print " - beanModule"
print " - connectionPoolModule"
print " - hamanagerModule"
print " - objectPoolModule "
class EsLogger(WorkerThread):
def __init__(self, threadName, workerConfig):
WorkerThread.__init__(self, threadName, workerConfig)
self.logger = Logger.getLogger("ElasticSearch.EsLogger")
# self.logger.setLevel(Level.DEBUG)
self.workerConfig = workerConfig
# Default Bulk Request Settings for ES Logging.
# - Set to True to use bulk requests for logs.
self.useBulkReq = False
self.bulkReqCounter = 0
self.bulkReqExecCountTrigger = 1000
self.lastBulkReqFlush = datetime.now()
if (threadName is None):
# ==== 1st Instance (threadName is None) ====
# Get the EsLogger queue.
# This object will feed the queue through this reference.
self.wq = workerConfig.wq
self.esNode = self.workerConfig.esNode
else:
# ==== 2nd Instance (threadName is not None) ====
self.esNode = self.workerConfig.esNode
self.esClient = self.esNode.getClient()
self.esBulkReq = EsBulkReq(self.esClient, None)
self.indexName = workerConfig.indexName
# If bulkReq config are set in the workerConfig object, use them.
if workerConfig.useBulkReq is not None:
self.useBulkReq = workerConfig.useBulkReq
if workerConfig.bulkReqExecCountTrigger is not None:
self.bulkReqExecCountTrigger = workerConfig.bulkReqExecCountTrigger
# Json SerDe objects
self.boon = BoonJson()
self.esLoggerWorker = None
self.esLoggerThread = None
self.stopThread = False
self.threaded = False
self.dtfmt = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ")
if threadName is None:
self.threadName = "EsLoggerController"
# Startup the Background thread.
self.startEsLoggerThread()
else:
self.threadName = threadName
def startEsLoggerThread(self):
if ( (self.esLoggerWorker is None)
and (self.esLoggerThread is None) ):
# Create the workers config object.
# Place the esLogger worker queue into the config object.
# Place the index name into the config object.
wconfig = EsLoggerConfig(self.esNode, self.workerConfig.esHosts,
self.workerConfig.indexName, self.wq)
wconfig.useBulkReq = self.workerConfig.useBulkReq
wconfig.bulkReqExecCountTrigger = self.workerConfig.bulkReqExecCountTrigger
if self.workerConfig.esNode is not None:
esNode = self.workerConfig.esNode
else:
esNode = self.esNode
if esNode == 0:
self.logger.error("An esNode is required to create the EsLogger WorkerThread.")
sys.exit(0)
wconfig.esNode = esNode
wconfig.esHosts = self.workerConfig.esHosts
# Create the esLogger object that will run on a thread.
# Give the object a thread name and the workerConfig object.
# This esLogger object will run on a thread and feed off of the
# worker queue in the worker config object.
self.esLoggerWorker = EsLogger("EsLoggerThread", wconfig)
# Create the thread.
self.esLoggerThread = Thread(name=self.esLoggerWorker.threadName,
target=self.esLoggerWorker.start, args=())
# Start the thread.
self.esLoggerThread.start()
else:
self.logger.warn("The EsLoggerThread is already running!")
def stopEsLoggerThread(self):
if self.esLoggerWorker.useBulkReq:
self.logger.debug("Flush Bulk Inserts...")
self.esLoggerWorker.flushBulkInserts()
# Give 7 seconds to flush any pending inserts.
self.logger.debug("-- Sleep 2...")
pytime.sleep(7)
if self.esLoggerWorker:
self.logger.debug("Stop Thread...")
self.esLoggerWorker.stopThread = True
# Allow 2 seconds for thread to die
self.logger.debug("-- Sleep 2...")
pytime.sleep(2)
# Use interrupts to stop threads.
# Interrupt is needed because the thread may be waiting on queue queue.
self.logger.debug("Interrupt Thread...")
tries = 5
while self.esLoggerWorker.isAlive() and tries > 0:
self.esLoggerWorker.interrupt()
pytime.sleep(.002)
tries = tries - 1
def stop(self):
self.stopEsLoggerThread()
def run(self):
self.threaded = True
self.wq = self.workerConfig.wq
while not self.stopThread:
# Read from workerQueue.
wrk = self.dequeueWork()
if wrk is not None:
# Log to ES
logLevel = wrk.get('logLevel')
self._logToEs(logLevel, wrk)
else:
# Flush bulk inserts if there are any in the que,
# Every 5 seconds.
unFlushedSecs = (datetime.now() - self.lastBulkReqFlush).seconds
if ( (unFlushedSecs >= 5)
and (self.bulkReqCounter > 0)):
self.logger.debug("...EsLogs Bulk Inserts... Flushing after 5 seconds...")
self.flushBulkInserts()
# Place work into the work queue.
def enqueueWork(self, workObj):
method = "enqueueWork"
self.logger.debug("[{}.{}] Place workObj into workerQueue".format(self.threadName,
method) )
self.wq.put(workObj)
# Pulls a work item from the queue.
def dequeueWork(self):
method = "dequeueWork"
# True: blocks if there are no items in the queue.
# 3: blocking on the queue releases after 1 seconds.
try:
wrk = None
wrk = self.wq.get(True,1)
except BaseException as ex:
# When we hit a BaseException while waiting on workQue, it is prob a timeout error on and empty queue.
if ('Empty' not in type(ex).__name__):
self.logger.error("BaseException in [{}.{}] - [{}: {}]".format(self.threadName, method, type(ex), ex))
if wrk is None:
self.logger.debug("--> Getting wrk, Queue.get() - timed out.")
except JException as ex:
self.logger.info("JException in [{}.{}] - [{}]".format(self.threadName, method, ex))
# If we get an interrupt which waiting for work, we need to stop the thread.
self.stopThread = True
return wrk
def logToEs(self, logLevel, jMap):
jMap.put('logDateTime', self.dtfmt.format(datetime.now()))
jMap.put('logLevel', logLevel)
# Push data into queue and return
self.enqueueWork(jMap)
# Accepts a Java Map object which we can serialize a json string via Boon.
# Then use the json string to store to ES.
def _logToEs(self, logLevel, jMap):
# *** Interactive Debug seems to mess with multi-threading
# pdb.set_trace()
# Convert the Java Map to a JSON string
jsonEvent = self.boon.serToJson(jMap)
try:
# Get the date from the logDateTime
edt = jMap.get('logDateTime')
# Pull the data from the TimeStamp
pat = r'^(\d+)\-(\d+)\-(\d+)T.*'
match = re.search(pat, edt)
if match:
edt = match.group(1) + match.group(2) + match.group(3)
else:
edt = 'unknown'
# Add the date to the index name.
indexName = self.indexName + '-' + edt
# Prep to Index document.
self.logger.debug("\n==index==> index[{}] logLevel[{}]\n".format(indexName,logLevel))
i1 = self.esClient.prepareIndex(indexName, logLevel, None)
i2 = i1.setSource(jsonEvent)
self.doEsInsert(i2, indexName, '', jsonEvent)
except JException as ex:
# ex: g.elasticsearch.index.mapper.MapperParsingException:...
# failed to parse [Data.Return.Value.Death.DateNormalized]
# Change the name of the bad field to s_<fieldName> in the jsonStr and reinsert the object.
# - Call on self.indexEvent(newJson) recursively as there may be multiple failures.
self.logger.error("EsLogger Exception Caught [{}] [{}]".format(ex, jsonEvent))
def doEsInsert(self, i2, indexName, typeMap, jsonEvent):
#=================================================================================
# Single index request operation...
# Submits one index operation at a time.
#=================================================================================
if not self.useBulkReq:
self.logger.debug("\n==single==> {}\n".format(jsonEvent))
# Execution a single index request.
indxResp = i2.execute().actionGet()
#=================================================================================
#=================================================================================
# Bulk Index Operation...
# Keep adding to the bulk index request until we reach N index operations,
# then submit the request.
#=================================================================================
else:
self.logger.debug("\n==bulk==> {}\n".format(jsonEvent))
self.esBulkReq.add(i2, indexName, typeMap, jsonEvent)
self.bulkReqCounter = self.bulkReqCounter + 1
if self.bulkReqCounter >= self.bulkReqExecCountTrigger:
self.esBulkReq.execute()
self.bulkReqCounter = 0
self.lastBulkReqFlush = datetime.now()
#=================================================================================
# If Bulk Request mode is in use, one should call on this function to flush
# any pending bulk requests at the end of bulk operation cycles.
def flushBulkInserts(self):
if self.useBulkReq and (self.bulkReqCounter > 0):
self.logger.debug("\n==> BulkReq Flush!\n")
self.esBulkReq.execute()
self.lastBulkReqFlush = datetime.now()
def close(self):
self.stopEsLoggerThread()
def currentDate(format = 'yyyy.MM.dd'): df = SimpleDateFormat(format) return df.format(Date())
#!/usr/bin/env python
from time import sleep
from java.util import Date
from java.text import SimpleDateFormat
username='******'
password='******'
adminurl='t3://host:port'
connect(username,password,adminurl)
domainRuntime()
servers=['server1','server2']
df=SimpleDateFormat('yyyy.MM.dd HH:mm:ss S')
fh = open('threads_mon.log', 'w')
while true:
now=df.format(Date())
for server in servers:
cd('ServerRuntimes/'+server+'/ThreadPoolRuntime/ThreadPoolRuntime')
compReq = cmo.getCompletedRequestCount()
status = cmo.getHealthState()
hoggingThreads = cmo.getHoggingThreadCount()
totalThreads = cmo.getExecuteThreadTotalCount()
idleThrds = cmo.getExecuteThreadIdleCount()
pending = cmo.getPendingUserRequestCount()
qLen = cmo.getQueueLength()
thruput = cmo.getThroughput()
outline="%s %s %3d %3d %3d %3d %6.2f\n" % (now,server,totalThreads,idleThrds,pending,qLen,thruput)
#print '%s %s %d %d %d %d %f' % (now,server,totalThreads,idleThrds,pending,qLen,thruput)
fh.write(outline)
sleep(15)
fh.flush()
disconnect()
def getActiveTransactions(self):
activeTransactions = []
whereClauses = ArrayList()
whereClauses.add("FUNCNAME IN ('SAPWL_TCODE_AGGREGATION','SAPWL_TCODE_AGGREGATION_COPY')");
result = self.executeQuery("TFDIR", whereClauses, "FUNCNAME")#@@CMD_PERMISION sap protocol execution
functionName = None
if result.next():
functionName = result.getString("FUNCNAME")
if functionName == None:
logger.warn('getActiveTransactions: active transaction function is not found')
return activeTransactions
day = self.__client.getProperty('from_date')
if day == None:
today = Date()
sfDate = SimpleDateFormat("yyyyMMdd")
day = sfDate.format(today)
elif day.find('/') != -1:
try:
sfDate = SimpleDateFormat("MM/dd/yyyy")
parsedDate = sfDate.parse(day)
sfDate = SimpleDateFormat("yyyyMMdd")
day = sfDate.format(parsedDate)
except:
logger.reportWarning('Failed to parse date ', day)
logger.debug('Parsed start date:', day)
logger.debug('Active transactions from data:', day)
mapTransactionToUsers = None
getUsers = Boolean.parseBoolean(self.__client.getProperty("get_users"))
if getUsers:
mapTransactionToUsers = HashMap()
funcParams = HashMap()
funcParams.put('READ_START_DATE', day)
funcParams.put('READ_START_TIME', '000000')
funcParams.put('READ_END_DATE', day)
funcParams.put('READ_END_TIME', '235959')
funcParams.put('READ_ONLY_MAINRECORDS', 'X')
logger.debug('executing func:SAPWL_STATREC_FROM_REMOTE_SYS(', str(funcParams),')')
fields = ArrayList()
fields.add('TCODE')
fields.add('ACCOUNT')
usersResult = self.__client.executeFunction('SAPWL_STATREC_FROM_REMOTE_SYS', funcParams, 'NORMAL_RECORDS', fields)
while usersResult.next():
transaction = usersResult.getString('TCODE')
if len(transaction) > 0:
user = usersResult.getString("ACCOUNT");
users = mapTransactionToUsers.get(transaction)
if users == None:
users = HashMap()
mapTransactionToUsers.put(transaction,users)
users.put(user,users);
self.getSites()
site = self.getSites().getCell(0,0)
servers = self.getServers(site)
numServers = servers.getRowCount()
transactionToStats = HashMap()
for j in range(numServers):
try:
instance = servers.getCell(j,0);
logger.debug('getActiveTransactions:executing function[' + functionName + '] for instance [' + instance + ']')
if functionName == 'SAPWL_TCODE_AGGREGATION_COPY':
records = self.callSapwlTcodeAggregationCopy(instance,day)
while records.next():
transaction = (str(records.getString(0))).strip()
mapUsers = None
if mapTransactionToUsers != None:
mapUsers = mapTransactionToUsers.get(transaction)
if (transaction != None) and (len(transaction) > 0):
stats = transactionToStats.get(transaction)
if stats == None:
stats = TransactionStatistics(transaction)
transactionToStats.put(transaction,stats)
if mapUsers != None:
stats.users = ArrayList(mapUsers.keySet())
if records.next():
stats.steps = stats.steps + int(float(records.getString(0)))
if records.next():
stats.responseTime = stats.responseTime + int(float(records.getString(0)))
if records.next():
stats.cpuTime = stats.cpuTime + int(float(records.getString(0)))
if records.next():
stats.dbTime = stats.dbTime + int(float(records.getString(0)))
if records.next():
stats.guiTime = stats.guiTime + int(float(records.getString(0)))
if records.next():
stats.roundTrips = stats.roundTrips + int(float(records.getString(0)))
if records.next():
stats.text = (str(records.getString(0))).strip()
else:
fields = ArrayList()
fields.add('ENTRY_ID')
fields.add('COUNT')
fields.add('RESPTI')
fields.add('CPUTI')
fields.add('DBTIME')
fields.add('GUITIME')
fields.add('GUICNT')
fields.add('TEXT')
records = self.getApplicationStatistics(functionName, instance, day, fields)
while records.next():
entryID = records.getString("ENTRY_ID");
transaction = self.getTransactionFromEntryID(entryID);
mapUsers = None
if mapTransactionToUsers != None:
mapUsers = mapTransactionToUsers.get(transaction)
if (transaction != None) and (len(transaction) > 0):
stats = transactionToStats.get(transaction)
if(stats == None):
stats = TransactionStatistics(transaction)
transactionToStats.put(transaction,stats)
if(mapUsers != None):
stats.users = ArrayList(mapUsers.keySet())
count = records.getString("COUNT")
stats.steps = stats.steps + int(count)
stats.responseTime = stats.responseTime + int(records.getString("RESPTI"))
stats.cpuTime = stats.cpuTime + int(records.getString("CPUTI"))
stats.dbTime = stats.dbTime + int(records.getString("DBTIME"))
stats.guiTime = stats.guiTime + int(records.getString("GUITIME"))
stats.roundTrips = stats.roundTrips + int(records.getString("GUICNT"))
stats.text = records.getString("TEXT")
except:
msg = sys.exc_info()[1]
strmsg = '%s' % msg
if strmsg.find('NO_DATA_FOUND') != -1:
logger.debug(strmsg)
logger.reportWarning('No data found in the given time range')
else:
logger.debugException('Unexpected error getting transactions for function:' + str(functionName))
logger.reportWarning('Unexpected error getting transactions for function:' + str(functionName) + ':' + strmsg)
transactions = ArrayList(transactionToStats.keySet())
logger.debug("getActiveTransactions: Found [" + str(transactions.size()) + "] active transactions")
if logger.isDebugEnabled():
logger.debug("getActiveTransactions: transactions = " + str(transactions))
transactionsInfo = self.getTransactionsInfo(transactions)
it = transactionToStats.values()
for stats in it:
prop = Properties()
prop.setProperty('data_name', str(stats.transaction))
prop.setProperty('dialog_steps', str(stats.steps))
prop.setProperty('total_response_time', str(stats.responseTime))
prop.setProperty('average_response_time', str(stats.getAverageCPUTime()))
prop.setProperty('total_cpu_time', str(stats.cpuTime))
prop.setProperty('average_cpu_time', str(stats.getAverageCPUTime()))
prop.setProperty('round_trips', str(stats.roundTrips))
prop.setProperty('total_db_time', str(stats.dbTime))
prop.setProperty('average_db_time', str(stats.getAverageDBTime()))
prop.setProperty('total_gui_time', str(stats.guiTime))
prop.setProperty('average_gui_time', str(stats.getAverageGUITime()))
prop.setProperty('text', stats.text)
prop.setProperty('saptransaction_averagedbtime', str(stats.users.size()))
info = transactionsInfo.get(stats.transaction)
if info != None:
prop.setProperty('devclass', info.devclass)
prop.setProperty('program', info.program)
prop.setProperty('screen', info.screen)
prop.setProperty('', info.screen)
else:
prop.setProperty('devclass', "")
prop.setProperty('program', "")
prop.setProperty('screen', "")
prop.setProperty('version', "")
activeTransactions.append(prop)
return activeTransactions
def getCurrentDate(self, format):
curDate = SimpleDateFormat(format)
return curDate.format(Date());
def getStdout(self):
return self.stdout.getOutput()
def getStdoutLines(self):
return self.stdout.getOutputLines()
def getStderr(self):
return self.stderr.getOutput()
def getStderrLines(self):
return self.stderr.getOutputLines()
myDate = Date(System.currentTimeMillis())
df = Sdf('yyyyMMddyyHHMMss')
dt = df.format(myDate)
testPath = testPath.replace( "\\", "\\\\")
testPath = testPath.replace( "\:", "\\\:")
paramfile = """
# Demo Parameters file
#
Test1=%s
resultsFilename=Results%s.xml
runType=FileSystem
PerScenarioTimeOut=%s
controllerPollingInterval=%s
fsTimeout=%s
""" % (testPath, dt, PerScenarioTimeout, controllerPolliingInterval, fsTimeout)
def formatVersion(self, dString):
dfSource = SimpleDateFormat("yyyyMMddHHmmss")
dfTarget = SimpleDateFormat("dd/MM/yyyy HH:mm:ss")
return dfTarget.format(dfSource.parse(dString))
def mustReader(paramMnemonics, startDate, endDate):
#
driverName = "com.mysql.jdbc.Driver"
#
#Class.forName(driverName)
#
url = "jdbc:mysql://" + Configuration.getProperty(
"vega.must.server.ip") + "/" + Configuration.getProperty(
"vega.must.server.repository"
) + "?user="******"vega.must.server.user"
) + "&password="******"vega.must.server.password")
#
loader = Thread.currentThread().getContextClassLoader()
c = loader.loadClass("com.mysql.jdbc.Driver")
#print c
driver = c.newInstance()
#print driver
con = driver.connect(url, java.util.Properties())
#con = DriverManager.getConnection(url)
stmt = con.createStatement()
formatter = SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
date1 = formatter.parse(endDate)
date2 = formatter.parse(startDate)
stringDate1 = formatter.format(date1)
stringDate2 = formatter.format(date2)
#
p = Product()
#ah=TableDataset()
p.meta["type"] = StringParameter(value="MUST Parameter Product")
p.meta["creator"] = StringParameter(value="MustClient")
p.meta["description"] = StringParameter(value="MUST")
p.meta["startDate"] = DateParameter(value=FineTime(date2))
p.meta["endDate"] = DateParameter(value=FineTime(date1))
#
for pars in range(len(paramMnemonics)):
#
ah = TableDataset()
parameterName = paramMnemonics[pars]
p.meta[parameterName] = BooleanParameter(value=Boolean(True))
#
sql = "select PID,DBTYPE from parameter where (PNAME='" + parameterName + "')"
rs = stmt.executeQuery(sql)
while (rs.next()):
pid = rs.getInt(1)
tableName = rs.getString(2) + "paramvalues"
sql = "select datetime,value from %s where (pid=%s and datetime<'%s' and datetime>'%s')" % (
tableName, pid, date1.time, date2.time)
rs = stmt.executeQuery(sql)
list = []
dates = Long1d()
values = Float1d()
while (rs.next()):
dates.append(rs.getLong(1))
values.append(rs.getFloat(2))
print "Retrieved parameter from MUST: ", parameterName
if (pars == 0):
#ah["Time"]=Column(data=dates,description="Time")
ah.meta["creator"] = StringParameter(value="MustClient")
ah.meta["description"] = StringParameter(value="MUST")
ah.meta["startDate"] = DateParameter(value=FineTime(date2))
ah.meta["endDate"] = DateParameter(value=FineTime(date1))
#
ah["Time"] = Column(data=dates, description="Time")
ah[parameterName] = Column(data=values, description=parameterName)
#
p[parameterName] = ah
rs.close()
stmt.close()
con.close()
#
return p
def date_as_string(date):
tz = TimeZone.getTimeZone("UTC")
df = SimpleDateFormat("yyyy-MM-dd HH:mm:ss")
df.setTimeZone(tz)
return df.format(date)
def mustReader(paramMnemonics,startDate,endDate) :
#
driverName="com.mysql.jdbc.Driver"
#
#Class.forName(driverName)
#
url = "jdbc:mysql://"+Configuration.getProperty("vega.must.server.ip")+"/"+Configuration.getProperty("vega.must.server.repository")+"?user="******"vega.must.server.user")+"&password="******"vega.must.server.password")
#
loader = Thread.currentThread().getContextClassLoader()
c = loader.loadClass("com.mysql.jdbc.Driver")
#print c
driver = c.newInstance()
#print driver
con = driver.connect(url, java.util.Properties())
#con = DriverManager.getConnection(url)
stmt = con.createStatement()
formatter = SimpleDateFormat ("yyyy-MM-dd HH:mm:ss")
date1 = formatter.parse(endDate)
date2 = formatter.parse(startDate)
stringDate1 = formatter.format(date1)
stringDate2 = formatter.format(date2)
#
p = Product()
#ah=TableDataset()
p.meta["type"]=StringParameter(value="MUST Parameter Product")
p.meta["creator"]=StringParameter(value="MustClient")
p.meta["description"]=StringParameter(value="MUST")
p.meta["startDate"]=DateParameter(value=FineTime(date2))
p.meta["endDate"]=DateParameter(value=FineTime(date1))
#
for pars in range(len(paramMnemonics)) :
#
ah=TableDataset()
parameterName = paramMnemonics[pars]
p.meta[parameterName]=BooleanParameter(value=Boolean(True))
#
sql = "select PID,DBTYPE from parameter where (PNAME='"+parameterName+"')"
rs = stmt.executeQuery(sql)
while (rs.next()):
pid=rs.getInt(1)
tableName = rs.getString(2)+"paramvalues"
sql = "select datetime,value from %s where (pid=%s and datetime<'%s' and datetime>'%s')"%(tableName,pid,date1.time,date2.time)
rs = stmt.executeQuery(sql)
list = []
dates=Long1d()
values=Float1d()
while (rs.next()):
dates.append(rs.getLong(1))
values.append(rs.getFloat(2))
print "Retrieved parameter from MUST: ", parameterName
if (pars == 0) :
#ah["Time"]=Column(data=dates,description="Time")
ah.meta["creator"]=StringParameter(value="MustClient")
ah.meta["description"]=StringParameter(value="MUST")
ah.meta["startDate"]=DateParameter(value=FineTime(date2))
ah.meta["endDate"]=DateParameter(value=FineTime(date1))
#
ah["Time"]=Column(data=dates,description="Time")
ah[parameterName]=Column(data=values,description=parameterName)
#
p[parameterName]=ah
rs.close()
stmt.close()
con.close()
#
return p
def cardData(context,
main=None,
add=None,
filterinfo=None,
session=None,
elementId=None):
u'''Функция данных для карточки редактирования содержимого таблицы разрешения. '''
gridContext = json.loads(
session)['sessioncontext']['related']['gridContext']
gridContext = gridContext if isinstance(gridContext,
list) else [gridContext]
currentId = {}
for gc in gridContext:
if "currentRecordId" in gc.keys():
currentId[gc["@id"]] = gc["currentRecordId"]
if add == 'add':
if "numbersSeriesGrid" in currentId.keys():
seriesId = currentId["numbersSeriesGrid"]
else:
seriesId = ''
xformsdata = {
"schema": {
"numberSeries": {
"@seriesId": seriesId,
"@numberOfLine": "",
"@startingDate": "",
"@startingNumber": "",
"@endingNumber": "",
"@incrimentByNumber": "",
"@lastUsedNumber": "",
"@isOpened": "",
"@lastUsedDate": "",
"@prefix": "",
"@postfix": "",
"@isFixedLength": ""
}
}
}
elif add == 'edit':
sdf = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS")
linesOfNumbersSeries = linesOfNumbersSeriesCursor(context)
linesOfNumbersSeries.get(currentId["numbersSeriesGrid"],
int(currentId["linesNumbersSeriesGrid"]))
xformsdata = {
"schema": {
"numberSeries": {
"@seriesId":
linesOfNumbersSeries.seriesId,
"@numberOfLine":
linesOfNumbersSeries.numberOfLine,
"@startingDate":
unicode(sdf.format(linesOfNumbersSeries.startingDate)),
"@startingNumber":
linesOfNumbersSeries.startingNumber,
"@endingNumber":
linesOfNumbersSeries.endingNumber,
"@incrimentByNumber":
linesOfNumbersSeries.incrimentByNumber,
"@lastUsedNumber":
linesOfNumbersSeries.lastUsedNumber,
"@isOpened":
unicode(linesOfNumbersSeries.isOpened).lower(),
"@lastUsedDate":
unicode(sdf.format(linesOfNumbersSeries.lastUsedDate)),
"@prefix":
linesOfNumbersSeries.prefix,
"@postfix":
linesOfNumbersSeries.postfix,
"@isFixedLength":
unicode(linesOfNumbersSeries.isFixedLength).lower()
}
}
}
# print xformsdata
xformssettings = {
"properties": {
"event": {
"@name": "single_click",
"@linkId": "1",
"action": {
"#sorted": [{
"main_context": "current"
}, {
"datapanel": {
"@type": "current",
"@tab": "current",
"element": {
"@id": "linesNumbersSeriesGrid",
"add_context": ""
}
}
}]
}
}
}
}
return JythonDTO(XMLJSONConverter.jsonToXml(json.dumps(xformsdata)),
XMLJSONConverter.jsonToXml(json.dumps(xformssettings)))
def formatDate(self, date):
dfSource = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss")
dfTarget = SimpleDateFormat("dd/MM/yyyy")
return dfTarget.format(dfSource.parse(date))
def format_not_after(self):
"""
Formats the expiration date/time of the certificate.
"""
df = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX")
return df.format(self.__not_after)
uv = DataMath.magnitude(uwnd, vwnd)
#uv = DataMath.Magnitude(uwnd, vwnd)
uvq = uv.mul(qv).div(9.8*1000)
#uvq = uv*qv/(9.8*1000)
#---- Create data layer
print 'Create data layer...'
dataLayer = DrawMeteoData.createShadedLayer(qhdivg, "WaterVaporFlux", "Flux", False)
#---- Add layer
print 'Add layers...'
mapFrame.addLayer(dataLayer)
mapFrame.moveLayer(dataLayer, 0)
#---- Zoom data
mapLayout.getActiveLayoutMap().zoomToExtentLonLatEx(Extent(0,360,-90.1,90.1))
#---- Set MapLayout
format = SimpleDateFormat('yyyy-MM-dd')
aTime = dataAir.getDataInfo().getTimes().get(tIdx)
mapLayout.addText('Water Vapor Flux Divergence - ' + format.format(aTime), 320, 30, 'Arial', 16)
aLegend = mapLayout.addLegend(650, 100)
aLegend.setLegendStyle(LegendStyles.Bar_Vertical)
aLegend.setLegendLayer(dataLayer)
layoutMap.setGridXDelt(60)
layoutMap.setGridYDelt(30)
layoutMap.setDrawGridLine(False)
mapLayout.paintGraphics()
frame = JFrame('MeteoInfo Script Sample', size = (750, 530))
frame.add(mapLayout)
frame.visible = True
print 'Finished!'
def formatDate(self, date):
dfSource = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss")
dfTarget = SimpleDateFormat("dd/MM/yyyy")
return dfTarget.format(dfSource.parse(date))
def formatVersion(self, dString):
dfSource = SimpleDateFormat("yyyyMMddHHmmss")
dfTarget = SimpleDateFormat("dd/MM/yyyy HH:mm:ss")
return dfTarget.format(dfSource.parse(dString))
def setupjob(job, args):
"""
Similar to the above, but run telemetry data that's already been exported
to HDFS.
Jobs expect two arguments, startdate and enddate, both in yyyyMMdd format.
"""
import java.text.SimpleDateFormat as SimpleDateFormat
import java.util.Date as Date
import java.util.Calendar as Calendar
import java.util.concurrent.TimeUnit as TimeUnit
import com.mozilla.util.DateUtil as DateUtil
import com.mozilla.util.DateIterator as DateIterator
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat as FileInputFormat
import org.apache.hadoop.mapreduce.lib.input.SequenceFileAsTextInputFormat as MyInputFormat
if len(args) != 2:
raise Exception("Usage: <startdate-YYYYMMDD> <enddate-YYYYMMDD>")
# use to collect up each date in the given range
class MyDateIterator(DateIterator):
def __init__(self):
self._list = []
def get(self):
return self._list
def see(self, aTime):
self._list.append(aTime)
sdf = SimpleDateFormat(dateformat)
sdf_hdfs = SimpleDateFormat(hdfs_dateformat)
startdate = Calendar.getInstance()
startdate.setTime(sdf.parse(args[0]))
enddate = Calendar.getInstance()
enddate.setTime(sdf.parse(args[1]))
nowdate = Calendar.getInstance()
# HDFS only contains the last 2 weeks of data (up to yesterday)
startMillis = startdate.getTimeInMillis()
endMillis = enddate.getTimeInMillis()
nowMillis = nowdate.getTimeInMillis()
startDiff = nowMillis - startMillis
if TimeUnit.DAYS.convert(startDiff, TimeUnit.MILLISECONDS) > 14:
raise Exception(
"HDFS Data only includes the past 14 days of history. Try again with more recent dates or use the HBase data directly."
)
endDiff = nowMillis - endMillis
if TimeUnit.DAYS.convert(endDiff, TimeUnit.MILLISECONDS) < 1:
raise Exception(
"HDFS Data only includes data up to yesterday. For (partial) data for today, use the HBase data directly."
)
dates = MyDateIterator()
DateUtil.iterateByDay(startMillis, endMillis, dates)
paths = []
for d in dates.get():
paths.append(hdfs_pathformat % (sdf_hdfs.format(Date(d))))
job.setInputFormatClass(MyInputFormat)
FileInputFormat.setInputPaths(job, ",".join(paths))
job.getConfiguration().set("org.mozilla.jydoop.mappertype", "TEXT")
def makeTimeSeriesContainer(tsData, timeZone, pathname=None):
'''
Construct a TimeSeriesContainer object from a python dictionary that was
created from a single "time-series" returned from the CWMS RADAR web
service
'''
#---------------#
# initial setup #
#---------------#
tsc = None
try:
tz = TimeZone.getTimeZone(timeZone)
sdf8601 = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX")
sdfHecTime = SimpleDateFormat("ddMMMyyyy, HH:mm")
cal = Calendar.getInstance()
for obj in sdf8601, sdfHecTime, cal:
obj.setTimeZone(tz)
ht = HecTime()
times, values, qualities = [], [], []
#------------------#
# process the data #
#------------------#
if tsData.has_key("regular-interval-values"):
#----------------------------------------#
# regular time series (a lot to process) #
#----------------------------------------#
rts = tsData["regular-interval-values"]
intvlStr = rts["interval"]
unit = rts["unit"].split()[0]
if intvlStr.startswith("PT"):
intvlNum, intvlUnit = int(intvlStr[2:-1]), intvlStr[-1]
try:
factor, field = {
"M": (1, Calendar.MINUTE),
"H": (60, Calendar.HOUR_OF_DAY)
}[intvlUnit]
except KeyError:
raise Exception("Unexpected interval: %s" % intvlStr)
else:
intvlNum, intvlUnit = int(intvlStr[1:-1]), intvlStr[-1]
try:
factor, field = {
"Y": (1440 * 365, Calendar.YEAR),
"M": (1440 * 30, Calendar.MONTH),
"D": (1440, Calendar.DATE)
}[intvlUnit]
except KeyError:
raise Exception("Unexpected interval: %s" % intvlStr)
intvl = intvlNum * factor
segmentCount = rts["segment-count"]
cal.setTimeInMillis(
sdf8601.parse(rts["segments"][0]["first-time"]).getTime())
for i in range(segmentCount):
for j in range(rts["segments"][i]["value-count"]):
ht.set(sdfHecTime.format(cal.getTimeInMillis()))
v, q = rts["segments"][i]["values"][j]
times.append(ht.value())
values.append(v)
qualities.append(q)
cal.add(field, intvlNum)
if i < segmentCount - 1:
nextBegin = sdf8601.parse(
rts["segments"][i + 1]["first-time"]).getTime()
time = cal.getTimeInMillis()
while time < nextBegin:
ht.set(sdfHecTime.format(time))
times.append(ht.value())
values.append(Constants.UNDEFINED)
qualities.append(0)
cal.add(field, intvlNum)
time = cal.getTimeInMillis()
elif tsData.has_key("irregular-interval-values"):
#------------------------------#
# irregular time series (easy) #
#------------------------------#
its = tsData["irregular-interval-values"]
unit = its["unit"].split()[0]
intvl = 0
for t, v, q in its["values"]:
ht.set(sdfHecTime.format(sdf8601.parse(t)))
times.append(ht.value())
values.append(v)
qualities.append(q)
else:
raise Exception("Time series has no values")
#--------------------------------------------------#
# code common to regular and irregular time series #
#--------------------------------------------------#
tsc = TimeSeriesContainer()
tsc.times = times
tsc.values = values
tsc.quality = qualities
tsc.numberValues = len(times)
tsc.startTime = times[0]
tsc.endTime = times[-1]
tsc.interval = intvl
tsc.units = unit
tsc.timeZoneID = timeZone
tsc.timeZoneRawOffset = tz.getRawOffset()
name = tsData["name"]
loc, param, paramType, intv, dur, ver = name.split(".")
if pathname:
#---------------------------#
# use pathname if specified #
#---------------------------#
A, B, C, D, E, F = 1, 2, 3, 4, 5, 6
parts = pathname.split("/")
parts[D] = ''
tsc.fullName = "/".join(parts)
tsc.watershed = parts[A]
try:
tsc.location, tsc.subLocation = parts[B].split("-", 1)
except:
tsc.location = parts[B]
try:
tsc.parameter, tsc.subParameter = parts[C].split("-", 1)
except:
tsc.parameter = parts[C]
try:
tsc.version, tsc.subVersion = parts[F].split("-", 1)
except:
tsc.version = parts[F]
else:
#--------------------------------------#
# no pathname, use CWMS time series id #
#--------------------------------------#
try:
tsc.location, tsc.subLocation = loc.split("-", 1)
except:
tsc.location = loc
try:
tsc.parameter, tsc.subParameter = param.split("-", 1)
except:
tsc.parameter = param
try:
tsc.version, tsc.subVersion = ver.split("-", 1)
except:
tsc.version = ver
tsc.type = {
"Total": "PER-CUM",
"Max": "PER-MAX",
"Min": "PER-MIN",
"Const": "INST-VAL",
"Ave": "PER-AVER",
"Inst": ("INST-VAL", "INST-CUM")[param.startswith("Precip")]
}[paramType]
except:
output(traceback.format_exc())
return tsc
def to_hectime(datestr, pattern="MM/dd/yyyy HH:mm"):
informat = SimpleDateFormat(pattern)
hecformat = SimpleDateFormat("ddMMMyyyy HHmm")
date = informat.parse(datestr)
hecdatestr = hecformat.format(date)
return HecTime(hecdatestr)
def _getTimeStamp():
# WAS style time stamp. SimpleDateFormat is not thread safe.
sdf = SimpleDateFormat("yy/MM/dd HH:mm:ss.sss z")
return '[' + sdf.format(Date()) + ']'
def listADDEImages(server, dataset, descriptor,
accounting=DEFAULT_ACCOUNTING,
location=None,
coordinateSystem=CoordinateSystems.LATLON,
place=None,
mag=None,
position=0,
unit=None,
day=None,
time=None,
debug=False,
band=None,
size=None):
"""Creates a list of ADDE images.
Args:
localEntry: Local ADDE dataset.
server: ADDE server.
dataset: ADDE dataset group name.
descriptor: ADDE dataset descriptor.
day: Day range. ('begin date', 'end date')
time: ('begin time', 'end time')
position: Position number. Values may be integers or the string "ALL". (default=0)
band: McIDAS band number; only images that have matching band number will be returned.
accounting: ('user', 'project number') User and project number required by servers using McIDAS accounting. default = ('idv','0')
Returns:
ADDE image matching the given criteria, if any.
"""
user = accounting[0]
proj = accounting[1]
debug = str(debug).lower()
if mag:
mag = '&MAG=%s %s' % (mag[0], mag[1])
else:
mag = ''
if unit:
unit = '&UNIT=%s' % (unit)
else:
unit = ''
if place is Places.CENTER:
place = '&PLACE=CENTER'
elif place is Places.ULEFT:
place = '&PLACE=ULEFT'
else:
# raise ValueError()
place = ''
if coordinateSystem is CoordinateSystems.LATLON:
coordSys = 'LATLON'
elif coordinateSystem is CoordinateSystems.AREA or coordinateSystem is CoordinateSystems.IMAGE:
coordSys = 'LINELE'
else:
raise ValueError()
if location:
location = '%s=%s %s' % (coordSys, location[0], location[1])
else:
location = ''
if size:
if size == 'ALL':
size = '&SIZE=99999 99999'
else:
size = '&SIZE=%s %s' % (size[0], size[1])
else:
size = ''
if time:
time = '&TIME=%s %s I' % (time[0], time[1])
else:
time = ''
if band:
band = '&BAND=%s' % (str(band))
else:
band = '&BAND=ALL'
tz = TimeZone.getTimeZone("Z")
dateFormat = SimpleDateFormat()
dateFormat.setTimeZone(tz)
dateFormat.applyPattern('yyyyDDD')
timeFormat = SimpleDateFormat();
timeFormat.setTimeZone(tz)
timeFormat.applyPattern('HH:mm:ss')
addeUrlFormat = "adde://%(server)s/imagedirectory?&PORT=112&COMPRESS=gzip&USER=%(user)s&PROJ=%(proj)s&VERSION=1&DEBUG=%(debug)s&TRACE=0&GROUP=%(dataset)s&DESCRIPTOR=%(descriptor)s%(band)s%(location)s%(place)s%(size)s%(unit)s%(mag)s%(day)s%(time)s&POS=%(position)s"
areaDirectories = []
dates = _normalizeDates(day)
if not dates:
formatValues = {
'server': server,
'user': user,
'proj': proj,
'debug': debug,
'dataset': dataset,
'descriptor': descriptor,
'band': band,
'location': location,
'place': place,
'size': size,
'unit': unit,
'mag': mag,
'day': '',
'time': time,
'position': position,
}
url = addeUrlFormat % formatValues
print url
adl = AreaDirectoryList(url)
dirs = adl.getSortedDirs()
for areaDirectory in dirs[0]:
areaDirectories.append(areaDirectory)
else:
for date in dates:
urlDate = '&DAY=%s' % (date)
formatValues = {
'server': server,
'user': user,
'proj': proj,
'debug': debug,
'dataset': dataset,
'descriptor': descriptor,
'band': band,
'location': location,
'place': place,
'size': size,
'unit': unit,
'mag': mag,
'day': urlDate,
'time': time,
'position': position,
}
url = addeUrlFormat % formatValues
print url
adl = AreaDirectoryList(url)
dirs = adl.getSortedDirs()
for areaDirectory in dirs[0]:
areaDirectories.append(areaDirectory)
temp = _AreaDirectoryList()
for i, d in enumerate(areaDirectories):
# print i, d.getBands(), d.getSensorType(), d.getCenterLatitude(), d.getCenterLongitude()
print i, d
nominalTime = d.getNominalTime()
tempDay = dateFormat.format(nominalTime, StringBuffer(), FieldPosition(0)).toString()
tempTime = timeFormat.format(nominalTime, StringBuffer(), FieldPosition(0)).toString()
tempBand = list(d.getBands())
if len(tempBand) == 1:
tempBand = tempBand[0]
else:
# raise Exception
pass
dt = {
'server': server,
'dataset': dataset,
'descriptor': descriptor,
'band': tempBand,
'debug': debug,
'accounting': accounting,
'day': tempDay,
'time': (tempTime, tempTime),
'coordinateSystem': CoordinateSystems.AREA,
'location': (d.getLines(), d.getElements()),
}
temp.append(dt)
return temp
def listADDEImages(localEntry=None,
server=None, dataset=None, descriptor=None,
accounting=DEFAULT_ACCOUNTING,
location=None,
coordinateSystem=CoordinateSystems.LATLON,
place=None,
mag=None,
position=None,
unit=None,
day=None,
time=None,
debug=False,
band=None,
size=None,
showUrls=True):
"""Creates a list of ADDE images.
Args:
localEntry: Local ADDE dataset.
server: ADDE server.
dataset: ADDE dataset group name.
descriptor: ADDE dataset descriptor.
day: Day range. ('begin date', 'end date')
time: ('begin time', 'end time')
position: Position number. Values may be integers or the string "ALL". (default=0)
band: McIDAS band number; only images that have matching band number will be returned.
accounting: ('user', 'project number') User and project number required by servers using McIDAS accounting. default = ('idv','0')
Returns:
ADDE image matching the given criteria, if any.
"""
if localEntry:
server = localEntry.getAddress()
dataset = localEntry.getGroup()
descriptor = localEntry.getDescriptor().upper()
elif (server is None) or (dataset is None) or (descriptor is None):
raise TypeError("must provide localEntry or server, dataset, and descriptor values.")
if server == "localhost" or server == "127.0.0.1":
port = EntryStore.getLocalPort()
else:
port = "112"
# server = '%s:%s' % (server, port)
user = accounting[0]
proj = accounting[1]
debug = str(debug).lower()
if mag:
mag = '&MAG=%s %s' % (mag[0], mag[1])
else:
mag = ''
if unit:
origUnit = unit
unit = '&UNIT=%s' % (unit)
else:
# origUnit = None
unit = ''
if place is Places.CENTER:
place = '&PLACE=CENTER'
elif place is Places.ULEFT:
place = '&PLACE=ULEFT'
else:
# raise ValueError()
place = ''
if coordinateSystem is CoordinateSystems.LATLON:
coordSys = 'LATLON'
elif coordinateSystem is CoordinateSystems.AREA or coordinateSystem is CoordinateSystems.IMAGE:
coordSys = 'LINELE'
else:
raise ValueError()
if location:
location = '&%s=%s %s' % (coordSys, location[0], location[1])
else:
location = ''
if size:
if size == 'ALL':
size = '&SIZE=99999 99999'
else:
size = '&SIZE=%s %s' % (size[0], size[1])
else:
size = ''
if time:
time = '&TIME=%s %s I' % (time[0], time[1])
else:
time = ''
if band:
band = '&BAND=%s' % (str(band))
else:
band = '&BAND=ALL'
if position is not None:
if isinstance(position, int):
position = '&POS=%s' % (position)
elif isinstance(position, tuple):
if len(position) != 2:
raise ValueError('position range may only contain values for the beginning and end of a range.')
position = '&POS=%s %s' % (str(position[0]), str(position[1]))
else:
position = '&POS=%s' % (str(position).upper())
else:
position = '&POS=0'
tz = TimeZone.getTimeZone('Z')
dateFormat = SimpleDateFormat()
dateFormat.setTimeZone(tz)
dateFormat.applyPattern('yyyyDDD')
timeFormat = SimpleDateFormat();
timeFormat.setTimeZone(tz)
timeFormat.applyPattern('HH:mm:ss')
addeUrlFormat = "adde://%(server)s/imagedirectory?&PORT=%(port)s&COMPRESS=gzip&USER=%(user)s&PROJ=%(proj)s&VERSION=1&DEBUG=%(debug)s&TRACE=0&GROUP=%(dataset)s&DESCRIPTOR=%(descriptor)s%(band)s%(location)s%(place)s%(size)s%(unit)s%(mag)s%(day)s%(time)s%(position)s"
urls = []
areaDirectories = []
dates = _normalizeDates(day)
for date in dates:
formatValues = {
'server': server,
'port': port,
'user': user,
'proj': proj,
'debug': debug,
'dataset': dataset,
'descriptor': descriptor,
'band': band,
'location': location,
'place': place,
'size': size,
'unit': unit,
'mag': mag,
'day': date,
'time': time,
'position': position,
}
url = addeUrlFormat % formatValues
if showUrls:
print url
adl = AreaDirectoryList(url)
results = adl.getSortedDirs()
for imageTimes in results:
for areaDirectory in imageTimes:
urls.append(url)
areaDirectories.append(areaDirectory)
temp = _AreaDirectoryList()
for i, d in enumerate(areaDirectories):
nominalTime = d.getNominalTime()
tempDay = str(dateFormat.format(nominalTime, StringBuffer(), FieldPosition(0)))
tempTime = str(timeFormat.format(nominalTime, StringBuffer(), FieldPosition(0)))
bandList = list(d.getBands())
# tempUnitList = list(d.getCalInfo()[0])
# unitList = tempUnitList[::2]
# unitDescList = tempUnitList[1::2]
# calInfo = dict(zip(unitList, unitDescList))
if unit:
unitList = [origUnit]
else:
unitList = map(str, list(d.getCalInfo()[0])[::2])
for band in bandList:
for calUnit in unitList:
dt = {
'server': server,
'dataset': dataset,
'descriptor': descriptor,
'bandNumber': band,
'bandList': bandList,
'debug': debug,
'accounting': accounting,
'day': tempDay,
'time': tempTime,
'imageSize': (d.getLines(), d.getElements()),
'centerLocation': (d.getCenterLatitude(), d.getCenterLongitude()),
'resolution': (d.getCenterLatitudeResolution(), d.getCenterLongitudeResolution()),
'unitList': unitList,
'unitType': calUnit,
'bands': bandList,
'band-count': d.getNumberOfBands(),
'calinfo': map(str, list(d.getCalInfo()[0])),
'calibration-scale-factor': d.getCalibrationScaleFactor(),
'calibration-type': str(d.getCalibrationType()),
'calibration-unit-name': d.getCalibrationUnitName(),
'center-latitude': d.getCenterLatitude(),
'center-latitude-resolution': d.getCenterLatitudeResolution(),
'center-longitude': d.getCenterLongitude(),
'center-longitude-resolution': d.getCenterLongitudeResolution(),
'directory-block': list(d.getDirectoryBlock()),
'elements': d.getElements(),
'lines': d.getLines(),
'memo-field': str(d.getMemoField()),
'nominal-time': DateTime(d.getNominalTime()),
'sensor-id': d.getSensorID(),
'sensor-type': str(d.getSensorType()),
'source-type': str(d.getSourceType()),
'start-time': DateTime(d.getStartTime()),
'url': urls[i],
}
temp.append(dt)
return temp
