def enhancePages( self, project, connector_url, cq_user, cq_password, page_query, page_queryLib ): # Validate Parameters if not self.validInputs(connector_url,cq_user,cq_password,"asset_drive","asset_filenameAttr","asset_pathAttr",page_query,page_queryLib,"ia_path","ia_cacheCols"): return # HTTP Connection Details client = HttpClient() base64EncodedCredentials = Base64.encodeString("%s:%s" % (cq_user,cq_password)) header = Header("Authorization", "Basic %s" % (base64EncodedCredentials)) # Cycle through contentdescriptors specified by scope query query = queryLibrarianService.findQueryByName(page_queryLib, page_query) params = HashMap() params.put('projectId',project.id) res = queryLibrarianService.executeQuery(query, params) # Get cd for r in res: cd = r[0] post = PostMethod(connector_url) post.addRequestHeader(header) get = GetMethod(connector_url) get.addRequestHeader(header) try: payload = cd.getContent()[0].getContentData().encode("utf-8") [loadStatus, failureReason] = self.sendPayload(client,get,post,payload,cd.url) cd.metadata["Load.Status"] = loadStatus cd.metadata["Load.Failure Reason"] = failureReason if loadStatus != "SUCCESS": logger.error("url: %s, reason: %s" % (cd.url, failureReason)) except: cd.metadata["Load.Status"] = "FAILURE" cd.metadata["Load.Failure Reason"] = "Null content" logger.error("url: %s, reason: %s" % (cd.url, failureReason))
def __get(self): try: get = GetMethod(self.url) statusInt = client.executeMethod(get) r = str(statusInt), get.getResponseBodyAsString().strip() except Exception, e: r = str(e), None
def enhanceIA( self, connector_url, cq_user, cq_password, ia_path, ia_cacheCols ): # Validate Parameters if not self.validInputs(connector_url,cq_user,cq_password,"asset_drive","asset_filenameAttr","asset_pathAttr","page_query","page_queryLib",ia_path,ia_cacheCols): return # HTTP Connection Details client = HttpClient() base64EncodedCredentials = Base64.encodeString("%s:%s" % (cq_user,cq_password)) header = Header("Authorization", "Basic %s" % (base64EncodedCredentials)) # IA Build cached_IA = self.cacheIA(ia_path,ia_cacheCols) for placeholder in cached_IA: post = PostMethod(connector_url) post.addRequestHeader(header) get = GetMethod(connector_url) get.addRequestHeader(header) # Prep Payload path = placeholder["Path"] filename = placeholder["Filename"] template_path = placeholder["Template Path"] label = placeholder["Label"] # Send Payload payload = '<page path="%s" filename="%s" template_path="%s" label="%s"><node path="jcr:content" type="nt:unstructured"><prop name="jcr:title" type="String">%s</prop></node></page>' % (path,filename,template_path,label,label) [loadStatus, failureReason] = self.sendPayload(client,get,post,payload,path+filename) if loadStatus != "SUCCESS": logger.error("url: %s, reason: %s" % (placeholder["URL"], failureReason))
def updateLocalRecordRelations(self, jobItems): oidIdentifierMap = HashMap() for jobItem in jobItems: oidIdentifierMap.put(jobItem.get("oid"),jobItem.get("required_identifiers")[0].get("identifier")) for jobItem in jobItems: type = jobItem.get("type"); targetSystem = self.systemConfig.getString(None, "curation", "supported-types", type); if targetSystem == "redbox": oid = jobItem.get("oid") digitalObject = StorageUtils.getDigitalObject(self.services.getStorage(), oid) tfPackagePid = self.getPackageData(digitalObject) metadataJsonPayload = digitalObject.getPayload(tfPackagePid) metadataJsonInstream = metadataJsonPayload.open() metadataJson = JsonSimple(metadataJsonInstream) metadataJsonPayload.close() relationships = metadataJson.getArray("relationships") if relationships is not None: for relationship in relationships: system = relationship.get("system") if system != "redbox" or system != None: url = self.systemConfig.getString("can't find it", "curation","external-system-urls","get-oid-for-identifier",system) client = BasicHttpClient(url+ "&identifier="+relationship.get("identifier")) get = GetMethod(url+ "&identifier="+relationship.get("identifier")) client.executeMethod(get) if get.getStatusCode() == 200: response = JsonSimple(get.getResponseBodyAsString()) relationship.put("curatedPid",oidIdentifierMap.get(response.getString(None,"oid"))) relationship.put("isCurated",True) #Now update the relationship on Mint's side break istream = ByteArrayInputStream(String(metadataJson.toString(True)).getBytes()) StorageUtils.createOrUpdatePayload(digitalObject,tfPackagePid,istream)
def __get(self): try: get = GetMethod(self.url) statusInt = self.client.executeMethod(get) r = str(statusInt), get.getResponseBodyAsString().strip() except Exception, e: r = str(e), None
def __get(self, url): try: client = BasicHttpClient(url) get = GetMethod(url) statusInt = client.executeMethod(get) s = get.getResponseBodyAsString().strip() except Exception, e: print "__get() Error - '%s'" % str(e) s = ""
def urlGet(self, url): try: client = BasicHttpClient(url) get = GetMethod(url) code = client.executeMethod(get) body = get.getResponseBodyAsString().strip() return (str(code), body) except Exception, e: self.log.error("Error during HTTP request: ", e) return (None, None)
def enhance( self, contentDescriptor, content, connector_url, cq_user, cq_password, asset_doWebDAV, asset_skeleton, asset_transformationRules, asset_drive, asset_filenameAttr, asset_pathAttr ): # Validate Parameters if not self.validInputs(connector_url,cq_user,cq_password,asset_drive,asset_filenameAttr,asset_pathAttr,"page_query","page_querylib","ia_path","ia_cacheCols"): return if asset_doWebDAV: # Save Asset to WebDAV Drive drive = asset_drive[0] path = re.sub("/content/dam/","/var/dam/",contentDescriptor.metadata[asset_pathAttr]) filename = contentDescriptor.metadata[asset_filenameAttr] filepath = "%s:%s/%s" % (drive,path,filename) if not exists("%s:%s" % (drive,path)): makedirs("%s:%s" % (drive,path)) file = BufferedOutputStream(FileOutputStream(filepath)) data = Base64.decode(content.contentData) try: file.write(data, 0, len(data)) file.close() contentDescriptor.metadata["Load.Status"] = "SUCCESS" except: logger.error("! [WebDAV] Could not write %s to file system" % (contentDescriptor.url)) file.close() contentDescriptor.metadata["Load.Status"] = "FAILURE" contentDescriptor.metadata["Load.Failure Reason"] = "! [WebDAV] Could not deposit file on file system" return # Build & Send Asset Metadata Payload if required if asset_skeleton and asset_transformationRules: # Build Payload payload = self.buildAssetPayload(asset_skeleton,asset_transformationRules,content,contentDescriptor) # HTTP Connection Details client = HttpClient() base64EncodedCredentials = Base64.encodeString("%s:%s" % (cq_user,cq_password)) header = Header("Authorization", "Basic %s" % (base64EncodedCredentials)) # Send Metadata Payload post = PostMethod(connector_url) post.addRequestHeader(header) get = GetMethod(connector_url) get.addRequestHeader(header) [loadStatus, failureReason] = self.sendPayload(client,get,post,payload,contentDescriptor.url) contentDescriptor.metadata["Load.Status"] = loadStatus if loadStatus == "SUCCESS": contentDescriptor.metadata["Load.Failure Reason"] = '' else: contentDescriptor.metadata["Load.Failure Reason"] = "Metadata: "+failureReason logger.error("! [METADATA] %s" % (failureReason))
def notifyExternalRelationship(self, relationship,pid,system,identifier): try: url = self.systemConfig.getString(None, "curation","external-system-urls","notify-curation",system) self.writer.println(url+ "&relationship=isCollectedBy&curatedPid="+pid+"&identifier="+relationship.get("identifier")+"&system="+self.systemConfig.getString(None,"system")+"&sourceIdentifier="+identifier) client = BasicHttpClient(url+ "&relationship=isCollectedBy&curatedPid="+pid+"&identifier="+relationship.get("identifier")+"&system="+self.systemConfig.getString(None,"system")+"&sourceIdentifier="+identifier) get = GetMethod(url+ "&relationship=isCollectedBy&curatedPid="+pid+"&identifier="+relationship.get("identifier")+"&system="+self.systemConfig.getString(None,"system")+"&sourceIdentifier="+identifier) client.executeMethod(get) status = get.getStatusCode() if status != 200: text = get.getStatusText() self.log.error(String.format("Error accessing ReDBox: %s %s",status, text)); return None; except Exception, ex: return None;
def doHttpGet(url, timeout=20000, requestedData='body', headerName=None): """ Performs HTTP(S) Connection to the specified URL Returns data according to the requestedData flag: 'body': Full Response Body as String 'header': Returns the response header with the specified headerName """ if requestedData == 'header': method = HeadMethod(url) else: method = GetMethod(url) client = HttpClient() client.getHttpConnectionManager().getParams().setConnectionTimeout(timeout) client.getHttpConnectionManager().getParams().setSoTimeout(timeout) client.executeMethod(method) if (requestedData == 'body'): return method.getResponseBodyAsString() elif (requestedData == 'header'): if headerName: return method.getResponseHeader(headerName) else: result = method.getResponseHeaders() return ''.join([s.toString() for s in result]) else: raise ValueError('Response part %s in not supported' % requestedData)
def notifyExternalRelationship(self,relationship,pid,system,identifier): try: url = self.systemConfig.getString(None, "curation","external-system-urls","notify-curation",system) url = "http://localhost:9001/default/api/notifyCuration.script?apiKey=1412412412241" completeUrl = url+ "&relationship=isCollectorOf&curatedPid="+pid+"&identifier="+relationship.get("identifier")+"&system="+self.systemConfig.getString("redbox","system")+"&sourceIdentifier="+identifier self.writer.println("the completeUrl: "+ completeUrl) client = BasicHttpClient(completeUrl) get = GetMethod(completeUrl) client.executeMethod(get) status = get.getStatusCode() if status != 200: text = get.getStatusText() self.log.error(String.format("Error accessing Mint",status, text)); return None; except Exception, ex: return None;
def enhancePages(self, project, connector_url, cq_user, cq_password, page_query, page_queryLib): # Validate Parameters if not self.validInputs(connector_url, cq_user, cq_password, "asset_drive", "asset_filenameAttr", "asset_pathAttr", page_query, page_queryLib, "ia_path", "ia_cacheCols"): return # HTTP Connection Details client = HttpClient() base64EncodedCredentials = Base64.encodeString("%s:%s" % (cq_user, cq_password)) header = Header("Authorization", "Basic %s" % (base64EncodedCredentials)) # Cycle through contentdescriptors specified by scope query query = queryLibrarianService.findQueryByName(page_queryLib, page_query) params = HashMap() params.put('projectId', project.id) res = queryLibrarianService.executeQuery(query, params) # Get cd for r in res: cd = r[0] post = PostMethod(connector_url) post.addRequestHeader(header) get = GetMethod(connector_url) get.addRequestHeader(header) try: payload = cd.getContent()[0].getContentData().encode("utf-8") [loadStatus, failureReason] = self.sendPayload(client, get, post, payload, cd.url) cd.metadata["Load.Status"] = loadStatus cd.metadata["Load.Failure Reason"] = failureReason if loadStatus != "SUCCESS": logger.error("url: %s, reason: %s" % (cd.url, failureReason)) except: cd.metadata["Load.Status"] = "FAILURE" cd.metadata["Load.Failure Reason"] = "Null content" logger.error("url: %s, reason: %s" % (cd.url, failureReason))
def queryJobStatus(self, curationJob): relations = ArrayList() get = None try: url = self.systemConfig.getString(None, "curation","curation-manager-url") client = BasicHttpClient(url + "/job/"+ curationJob.getCurationJobId()) get = GetMethod(url+ "/job/"+ curationJob.getCurationJobId()) client.executeMethod(get) status = get.getStatusCode() if status != 200: text = get.getStatusText() self.log.error(String.format("Error accessing Curation Manager, status code '%d' returned with message: %s",status, text)); return None; except Exception, ex: return None;
def __init__(self): responseType = "text/html; charset=UTF-8" responseMsg = "" func = formData.get("func") if func == "placeName": try: placeName = formData.get("q") url = "http://ws.geonames.org/searchJSON?fuzzy=0.7&q=" + placeName client = BasicHttpClient(url) get = GetMethod(url) statusInt = client.executeMethod(get) r = str(statusInt) jsonConfigHelper = JsonConfigHelper(get.getResponseBodyAsString().strip()) for geoName in jsonConfigHelper.getJsonList("geonames"): responseMsg += "%s, %s|%s \n" % (geoName.get("name"), geoName.get("countryName"), geoName.get("geonameId")) except Exception, e: print "exception: ", str(e) r = str(e), None responseType = "text/plain; charset=UTF-8"
def enhanceIA(self, connector_url, cq_user, cq_password, ia_path, ia_cacheCols): # Validate Parameters if not self.validInputs(connector_url, cq_user, cq_password, "asset_drive", "asset_filenameAttr", "asset_pathAttr", "page_query", "page_queryLib", ia_path, ia_cacheCols): return # HTTP Connection Details client = HttpClient() base64EncodedCredentials = Base64.encodeString("%s:%s" % (cq_user, cq_password)) header = Header("Authorization", "Basic %s" % (base64EncodedCredentials)) # IA Build cached_IA = self.cacheIA(ia_path, ia_cacheCols) for placeholder in cached_IA: post = PostMethod(connector_url) post.addRequestHeader(header) get = GetMethod(connector_url) get.addRequestHeader(header) # Prep Payload path = placeholder["Path"] filename = placeholder["Filename"] template_path = placeholder["Template Path"] label = placeholder["Label"] # Send Payload payload = '<page path="%s" filename="%s" template_path="%s" label="%s"><node path="jcr:content" type="nt:unstructured"><prop name="jcr:title" type="String">%s</prop></node></page>' % ( path, filename, template_path, label, label) [loadStatus, failureReason] = self.sendPayload(client, get, post, payload, path + filename) if loadStatus != "SUCCESS": logger.error("url: %s, reason: %s" % (placeholder["URL"], failureReason))
def __activate__(self, context): self.vc = context responseType = "text/html; charset=UTF-8" responseMsg = "" func = self.vc("formData").get("func") if func == "placeName": try: placeName = self.vc("formData").get("q") url = "http://ws.geonames.org/searchJSON?fuzzy=0.7&q=" + placeName client = BasicHttpClient(url) get = GetMethod(url) statusInt = client.executeMethod(get) r = str(statusInt) json = JsonSimple(get.getResponseBodyAsString().strip()) for geoName in json.getJsonSimpleList("geonames"): responseMsg += "%s, %s|%s \n" % (geoName.getString(None, "name"), geoName.getString(None, "countryName"), geoName.getString(None, "geonameId")) except Exception, e: print "exception: ", str(e) r = str(e), None responseType = "text/plain; charset=UTF-8"
def saveHM(filename): # Create new HTTP client client = HttpClient() client.getParams().setAuthenticationPreemptive(True) defaultcreds = UsernamePasswordCredentials(user, password) client.getState().setCredentials(AuthScope.ANY, defaultcreds) # Get data across HTTP url = ( "http://" + host + ":" + str(port) + "/admin/savedataview.egi?type=" + type + "&data_format=NEXUSHDF5_LZW_6&data_saveopen_action=OPEN_ONLY" ) getMethod = GetMethod(url) getMethod.setDoAuthentication(True) client.executeMethod(getMethod) # Save locally file = File(directory + "/" + filename) out = FileOutputStream(file) out.write(getMethod.getResponseBody()) out.close() # Clean up getMethod.releaseConnection()
def queryJobStatus(self, curationJob): relations = ArrayList() get = None try: url = self.systemConfig.getString(None, "curation", "curation-manager-url") client = BasicHttpClient(url + "/job/" + curationJob.getCurationJobId()) get = GetMethod(url + "/job/" + curationJob.getCurationJobId()) client.executeMethod(get) status = get.getStatusCode() if status != 200: text = get.getStatusText() self.log.error( String.format( "Error accessing Curation Manager, status code '%d' returned with message: %s", status, text)) return None except Exception, ex: return None
def notifyExternalRelationship(self, relationship, pid, system, identifier): try: url = self.systemConfig.getString(None, "curation", "external-system-urls", "notify-curation", system) url = "http://localhost:9001/default/api/notifyCuration.script?apiKey=1412412412241" completeUrl = url + "&relationship=isCollectorOf&curatedPid=" + pid + "&identifier=" + relationship.get( "identifier") + "&system=" + self.systemConfig.getString( "redbox", "system") + "&sourceIdentifier=" + identifier self.writer.println("the completeUrl: " + completeUrl) client = BasicHttpClient(completeUrl) get = GetMethod(completeUrl) client.executeMethod(get) status = get.getStatusCode() if status != 200: text = get.getStatusText() self.log.error( String.format("Error accessing Mint", status, text)) return None except Exception, ex: return None
def __activate__(self, context): self.vc = context responseType = "text/html; charset=UTF-8" responseMsg = "" func = self.vc("formData").get("func") if func == "placeName": try: placeName = self.vc("formData").get("q") url = "http://ws.geonames.org/searchJSON?fuzzy=0.7&q=" + placeName client = BasicHttpClient(url) get = GetMethod(url) statusInt = client.executeMethod(get) r = str(statusInt) json = JsonSimple(get.getResponseBodyAsString().strip()) for geoName in json.getJsonSimpleList("geonames"): responseMsg += "%s, %s|%s \n" % (geoName.getString( None, "name"), geoName.getString(None, "countryName"), geoName.getString( None, "geonameId")) except Exception, e: print "exception: ", str(e) r = str(e), None responseType = "text/plain; charset=UTF-8"
def __wget(self, url): client = BasicHttpClient(url) m = GetMethod(url) client.executeMethod(m) return IOUtils.toString(m.getResponseBodyAsStream(), "UTF-8")
def enhance(self, contentDescriptor, content, connector_url, cq_user, cq_password, asset_doWebDAV, asset_skeleton, asset_transformationRules, asset_drive, asset_filenameAttr, asset_pathAttr): # Validate Parameters if not self.validInputs(connector_url, cq_user, cq_password, asset_drive, asset_filenameAttr, asset_pathAttr, "page_query", "page_querylib", "ia_path", "ia_cacheCols"): return if asset_doWebDAV: # Save Asset to WebDAV Drive drive = asset_drive[0] path = re.sub("/content/dam/", "/var/dam/", contentDescriptor.metadata[asset_pathAttr]) filename = contentDescriptor.metadata[asset_filenameAttr] filepath = "%s:%s/%s" % (drive, path, filename) if not exists("%s:%s" % (drive, path)): makedirs("%s:%s" % (drive, path)) file = BufferedOutputStream(FileOutputStream(filepath)) data = Base64.decode(content.contentData) try: file.write(data, 0, len(data)) file.close() contentDescriptor.metadata["Load.Status"] = "SUCCESS" except: logger.error("! [WebDAV] Could not write %s to file system" % (contentDescriptor.url)) file.close() contentDescriptor.metadata["Load.Status"] = "FAILURE" contentDescriptor.metadata[ "Load.Failure Reason"] = "! [WebDAV] Could not deposit file on file system" return # Build & Send Asset Metadata Payload if required if asset_skeleton and asset_transformationRules: # Build Payload payload = self.buildAssetPayload(asset_skeleton, asset_transformationRules, content, contentDescriptor) # HTTP Connection Details client = HttpClient() base64EncodedCredentials = Base64.encodeString( "%s:%s" % (cq_user, cq_password)) header = Header("Authorization", "Basic %s" % (base64EncodedCredentials)) # Send Metadata Payload post = PostMethod(connector_url) post.addRequestHeader(header) get = GetMethod(connector_url) get.addRequestHeader(header) [loadStatus, failureReason] = self.sendPayload(client, get, post, payload, contentDescriptor.url) contentDescriptor.metadata["Load.Status"] = loadStatus if loadStatus == "SUCCESS": contentDescriptor.metadata["Load.Failure Reason"] = '' else: contentDescriptor.metadata[ "Load.Failure Reason"] = "Metadata: " + failureReason logger.error("! [METADATA] %s" % (failureReason))
import org.apache.commons.httpclient.*; import org.apache.commons.httpclient.methods.GetMethod; import org.apache.commons.httpclient.params.HttpMethodParams; public class Csdn { @SuppressWarnings("deprecation") public static void main(String[] args) { HttpClient httpClient = new HttpClient(); GetMethod getMethod = new GetMethod("http://hi.csdn.net/gxiangzi"); getMethod.getParams().setParameter(HttpMethodParams.RETRY_HANDLER, new DefaultMethodRetryHandler()); for (int i = 1; i <= 25; i++) { try { int statusCode = httpClient.executeMethod(getMethod); if (statusCode != HttpStatus.SC_OK) { System.out.print("失败:" + getMethod.getStatusLine()); } // byte[] responseBody=getMethod.getResponseBody();获取网页内容 // System.out.print(new String(responseBody)); } catch (Exception e) { System.out.print("请检查网络地址!"); } finally { getMethod.releaseConnection(); } System.out.println("您已经成功刷新了"+i+"次!"); } } }